Merge branch 'feature/GlobalRefactoring' into develop

.gitignore

@@ -1,11 +1,12 @@
 # project data
-data/*
-saved_models/*
-threshold_map/*
-models_info/*
-custom_norm/*
-learned_zones/*
-corr_indices/*
+data
+saved_models
+threshold_map
+models_info
+custom_norm
+learned_zones
+corr_indices
+metric_curves
 .ipynb_checkpoints
 
 # simulate_models.csv
@@ -16,7 +17,6 @@ dataset
 __pycache__
 
 # by default avoid model files and png files
-saved_models/*.h5
 *.png
 !saved_models/*.png
 .vscode

README.md

@@ -17,10 +17,10 @@ Filters list:
 pip install -r requirements.txt
 ```
 
-Generate all needed data for each metrics (which requires the the whole dataset. In order to get it, you need to contact us).
+Generate all needed data for each features (which requires the the whole dataset. In order to get it, you need to contact us).
 
 ```bash
-python generate_all_data.py --metric all
+python generate/generate_all_data.py --feature all
 ```
 
 ## How to use
@@ -68,19 +68,19 @@ Expected values for the **choice** parameter are ['svm_model', 'ensemble_model',
 Now we have a model trained, we can use it with an image as input:
 
 ```bash
-python predict_noisy_image_svd.py --image path/to/image.png --interval "x,x" --model saved_models/xxxxxx.joblib --metric 'lab' --mode 'svdn' --custom 'min_max_filename'
+python prediction/predict_noisy_image_svd.py --image path/to/image.png --interval "x,x" --model saved_models/xxxxxx.joblib --feature 'lab' --mode 'svdn' --custom 'min_max_filename'
 ```
 
-- **metric**: metric choice need to be one of the listed above.
+- **feature**: feature choice need to be one of the listed above.
 - **custom**: specify filename with custom min and max from your data interval. This file was generated using **custom** parameter of one of the **generate_data_model\*.py** script (optional parameter).
 
 The model will return only 0 or 1:
 - 1 means noisy image is detected.
 - 0 means image seem to be not noisy.
 
-All SVD metrics developed need:
-- Name added into *metric_choices_labels* global array variable of **modules/utils/config.py** file.
-- A specification of how you compute the metric into *get_svd_data* method of **modules/utils/data_type.py** file.
+All SVD features developed need:
+- Name added into *feature_choices_labels* global array variable of **modules/utils/config.py** file.
+- A specification of how you compute the feature into *get_svd_data* method of **modules/utils/data_type.py** file.
 
 ### Predict scene using model
 
@@ -93,13 +93,13 @@ python prediction_scene.py --data path/to/xxxx.csv --model saved_model/xxxx.jobl
 
 ### Visualize data
 
-All scripts with names **display_\*.py** are used to display data information or results.
+All scripts with names **display/display_\*.py** are used to display data information or results.
 
 Just use --help option to get more information.
 
 ### Simulate model on scene
 
-All scripts named **predict_seuil_expe\*.py** are used to simulate model prediction during rendering process. Do not forget the **custom** parameter filename if necessary.
+All scripts named **prediction/predict_seuil_expe\*.py** are used to simulate model prediction during rendering process. Do not forget the **custom** parameter filename if necessary.
 
 Once you have simulation done. Checkout your **threshold_map/%MODEL_NAME%/simulation\_curves\_zones\_\*/** folder and use it with help of **display_simulation_curves.py** script.
 

custom_config.py

@@ -0,0 +1,19 @@
+from modules.config.attributes_config import *
+
+# store all variables from global config
+context_vars = vars()
+
+# folders
+## min_max_custom_folder           = 'custom_norm'
+## correlation_indices_folder      = 'corr_indices'
+
+# variables
+features_choices_labels         = ['filters_statistics']
+
+## models_names_list               = ["svm_model","ensemble_model","ensemble_model_v2","deep_keras"]
+## normalization_choices           = ['svd', 'svdn', 'svdne']
+
+# parameters
+## keras_epochs                    = 500
+## keras_batch                     = 32
+## val_dataset_size                = 0.2

data_attributes.py

@@ -0,0 +1,464 @@
+# main imports
+import numpy as np
+import sys
+
+# image transform imports
+from PIL import Image
+from skimage import color
+from sklearn.decomposition import FastICA
+from sklearn.decomposition import IncrementalPCA
+from sklearn.decomposition import TruncatedSVD
+from numpy.linalg import svd as lin_svd
+from scipy.signal import medfilt2d, wiener, cwt
+import pywt
+import cv2
+
+from ipfml.processing import transform, compression, segmentation
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
+
+def get_svd_data(data_type, block):
+    """
+    Method which returns the data type expected
+    """
+
+    if data_type == 'lab':
+
+        block_file_path = '/tmp/lab_img.png'
+        block.save(block_file_path)
+        data = transform.get_LAB_L_SVD_s(Image.open(block_file_path))
+
+    if data_type == 'mscn':
+
+        img_mscn_revisited = transform.rgb_to_mscn(block)
+
+        # save tmp as img
+        img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
+        mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
+        img_output.save(mscn_revisited_file_path)
+        img_block = Image.open(mscn_revisited_file_path)
+
+        # extract from temp image
+        data = compression.get_SVD_s(img_block)
+
+    """if data_type == 'mscn':
+
+        img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
+        img_mscn = transform.calculate_mscn_coefficients(img_gray, 7)
+        img_mscn_norm = transform.normalize_2D_arr(img_mscn)
+
+        img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
+
+        data = compression.get_SVD_s(img_mscn_gray)
+    """
+
+    if data_type == 'low_bits_6':
+
+        low_bits_6 = transform.rgb_to_LAB_L_low_bits(block, 6)
+        data = compression.get_SVD_s(low_bits_6)
+
+    if data_type == 'low_bits_5':
+
+        low_bits_5 = transform.rgb_to_LAB_L_low_bits(block, 5)
+        data = compression.get_SVD_s(low_bits_5)
+
+    if data_type == 'low_bits_4':
+
+        low_bits_4 = transform.rgb_to_LAB_L_low_bits(block, 4)
+        data = compression.get_SVD_s(low_bits_4)
+
+    if data_type == 'low_bits_3':
+
+        low_bits_3 = transform.rgb_to_LAB_L_low_bits(block, 3)
+        data = compression.get_SVD_s(low_bits_3)
+
+    if data_type == 'low_bits_2':
+
+        low_bits_2 = transform.rgb_to_LAB_L_low_bits(block, 2)
+        data = compression.get_SVD_s(low_bits_2)
+
+    if data_type == 'low_bits_4_shifted_2':
+
+        data = compression.get_SVD_s(transform.rgb_to_LAB_L_bits(block, (3, 6)))
+
+    if data_type == 'sub_blocks_stats':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 4), int(height / 4)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+
+            # get information we want from svd
+            data.append(np.mean(l_svd_data))
+            data.append(np.median(l_svd_data))
+            data.append(np.percentile(l_svd_data, 25))
+            data.append(np.percentile(l_svd_data, 75))
+            data.append(np.var(l_svd_data))
+
+            area_under_curve = utils.integral_area_trapz(l_svd_data, dx=100)
+            data.append(area_under_curve)
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'sub_blocks_stats_reduced':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 4), int(height / 4)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+
+            # get information we want from svd
+            data.append(np.mean(l_svd_data))
+            data.append(np.median(l_svd_data))
+            data.append(np.percentile(l_svd_data, 25))
+            data.append(np.percentile(l_svd_data, 75))
+            data.append(np.var(l_svd_data))
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'sub_blocks_area':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 8), int(height / 8)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+
+            area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
+            data.append(area_under_curve)
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'sub_blocks_area_normed':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 8), int(height / 8)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = utils.normalize_arr(l_svd_data)
+
+            area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
+            data.append(area_under_curve)
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'mscn_var_4':
+
+        data = _get_mscn_variance(block, (100, 100))
+
+    if data_type == 'mscn_var_16':
+
+        data = _get_mscn_variance(block, (50, 50))
+
+    if data_type == 'mscn_var_64':
+
+        data = _get_mscn_variance(block, (25, 25))
+
+    if data_type == 'mscn_var_16_max':
+
+        data = _get_mscn_variance(block, (50, 50))
+        data = np.asarray(data)
+        size = int(len(data) / 4)
+        indices = data.argsort()[-size:][::-1]
+        data = data[indices]
+
+    if data_type == 'mscn_var_64_max':
+
+        data = _get_mscn_variance(block, (25, 25))
+        data = np.asarray(data)
+        size = int(len(data) / 4)
+        indices = data.argsort()[-size:][::-1]
+        data = data[indices]
+
+    if data_type == 'ica_diff':
+        current_image = transform.get_LAB_L(block)
+
+        ica = FastICA(n_components=50)
+        ica.fit(current_image)
+
+        image_ica = ica.fit_transform(current_image)
+        image_restored = ica.inverse_transform(image_ica)
+
+        final_image = utils.normalize_2D_arr(image_restored)
+        final_image = np.array(final_image * 255, 'uint8')
+
+        sv_values = utils.normalize_arr(compression.get_SVD_s(current_image))
+        ica_sv_values = utils.normalize_arr(compression.get_SVD_s(final_image))
+
+        data = abs(np.array(sv_values) - np.array(ica_sv_values))
+
+    if data_type == 'svd_trunc_diff':
+
+        current_image = transform.get_LAB_L(block)
+
+        svd = TruncatedSVD(n_components=30, n_iter=100, random_state=42)
+        transformed_image = svd.fit_transform(current_image)
+        restored_image = svd.inverse_transform(transformed_image)
+
+        reduced_image = (current_image - restored_image)
+
+        U, s, V = compression.get_SVD(reduced_image)
+        data = s
+
+    if data_type == 'ipca_diff':
+
+        current_image = transform.get_LAB_L(block)
+
+        transformer = IncrementalPCA(n_components=20, batch_size=25)
+        transformed_image = transformer.fit_transform(current_image)
+        restored_image = transformer.inverse_transform(transformed_image)
+
+        reduced_image = (current_image - restored_image)
+
+        U, s, V = compression.get_SVD(reduced_image)
+        data = s
+
+    if data_type == 'svd_reconstruct':
+
+        reconstructed_interval = (90, 200)
+        begin, end = reconstructed_interval
+
+        lab_img = transform.get_LAB_L(block)
+        lab_img = np.array(lab_img, 'uint8')
+
+        U, s, V = lin_svd(lab_img, full_matrices=True)
+
+        smat = np.zeros((end-begin, end-begin), dtype=complex)
+        smat[:, :] = np.diag(s[begin:end])
+        output_img = np.dot(U[:, begin:end],  np.dot(smat, V[begin:end, :]))
+
+        output_img = np.array(output_img, 'uint8')
+
+        data = compression.get_SVD_s(output_img)
+
+    if 'sv_std_filters' in data_type:
+
+        # convert into lab by default to apply filters
+        lab_img = transform.get_LAB_L(block)
+        arr = np.array(lab_img)
+        images = []
+        
+        # Apply list of filter on arr
+        images.append(medfilt2d(arr, [3, 3]))
+        images.append(medfilt2d(arr, [5, 5]))
+        images.append(wiener(arr, [3, 3]))
+        images.append(wiener(arr, [5, 5]))
+        
+        # By default computation of current block image
+        s_arr = compression.get_SVD_s(arr)
+        sv_vector = [s_arr]
+
+        # for each new image apply SVD and get SV 
+        for img in images:
+            s = compression.get_SVD_s(img)
+            sv_vector.append(s)
+            
+        sv_array = np.array(sv_vector)
+        
+        _, len = sv_array.shape
+        
+        sv_std = []
+        
+        # normalize each SV vectors and compute standard deviation for each sub vectors
+        for i in range(len):
+            sv_array[:, i] = utils.normalize_arr(sv_array[:, i])
+            sv_std.append(np.std(sv_array[:, i]))
+        
+        indices = []
+
+        if 'lowest' in data_type:
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
+
+        if 'highest' in data_type:
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
+
+        # data are arranged following std trend computed
+        data = s_arr[indices]
+
+    # with the use of wavelet
+    if 'wave_sv_std_filters' in data_type:
+
+        # convert into lab by default to apply filters
+        lab_img = transform.get_LAB_L(block)
+        arr = np.array(lab_img)
+        images = []
+        
+        # Apply list of filter on arr
+        images.append(medfilt2d(arr, [3, 3]))
+        images.append(medfilt2d(arr, [5, 5]))
+        images.append(medfilt2d(arr, [7, 7]))
+        images.append(wiener(arr, [3, 3]))
+        images.append(wiener(arr, [4, 4]))
+        images.append(wiener(arr, [5, 5]))
+        images.append(w2d(arr, 'haar', 2))
+        images.append(w2d(arr, 'haar', 3))
+        images.append(w2d(arr, 'haar', 4))
+        
+        # By default computation of current block image
+        s_arr = compression.get_SVD_s(arr)
+        sv_vector = [s_arr]
+
+        # for each new image apply SVD and get SV 
+        for img in images:
+            s = compression.get_SVD_s(img)
+            sv_vector.append(s)
+            
+        sv_array = np.array(sv_vector)
+        
+        _, len = sv_array.shape
+        
+        sv_std = []
+        
+        # normalize each SV vectors and compute standard deviation for each sub vectors
+        for i in range(len):
+            sv_array[:, i] = utils.normalize_arr(sv_array[:, i])
+            sv_std.append(np.std(sv_array[:, i]))
+        
+        indices = []
+
+        if 'lowest' in data_type:
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
+
+        if 'highest' in data_type:
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
+
+        # data are arranged following std trend computed
+        data = s_arr[indices]
+
+    if 'filters_statistics' in data_type:
+
+        img_width, img_height = 200, 200
+
+        lab_img = transform.get_LAB_L(block)
+        arr = np.array(lab_img)
+
+        # compute all filters statistics
+        def get_stats(arr, I_filter):
+
+            e1       = np.abs(arr - I_filter)
+            L        = np.array(e1)
+            mu0      = np.mean(L)
+            A        = L - mu0
+            H        = A * A
+            E        = np.sum(H) / (img_width * img_height)
+            P        = np.sqrt(E)
+
+            return mu0, P
+
+        stats = []
+
+        kernel = np.ones((3,3),np.float32)/9
+        stats.append(get_stats(arr, cv2.filter2D(arr,-1,kernel)))
+
+        kernel = np.ones((5,5),np.float32)/25
+        stats.append(get_stats(arr, cv2.filter2D(arr,-1,kernel)))
+
+        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (3, 3), 0.5)))
+
+        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (3, 3), 1)))
+
+        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (3, 3), 1.5)))
+
+        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (5, 5), 0.5)))
+
+        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (5, 5), 1)))
+
+        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (5, 5), 1.5)))
+
+        stats.append(get_stats(arr, medfilt2d(arr, [3, 3])))
+
+        stats.append(get_stats(arr, medfilt2d(arr, [5, 5])))
+
+        stats.append(get_stats(arr, wiener(arr, [3, 3])))
+
+        stats.append(get_stats(arr, wiener(arr, [5, 5])))
+
+        wave = w2d(arr, 'db1', 2)
+        stats.append(get_stats(arr, np.array(wave, 'float64')))
+
+        data = []
+
+        for stat in stats:
+            data.append(stat[0])
+
+        for stat in stats:
+            data.append(stat[1])
+        
+        data = np.array(data)
+
+    return data
+
+
+def w2d(arr, mode='haar', level=1):
+    #convert to float   
+    imArray = arr
+    np.divide(imArray, 255)
+
+    # compute coefficients 
+    coeffs=pywt.wavedec2(imArray, mode, level=level)
+
+    #Process Coefficients
+    coeffs_H=list(coeffs)  
+    coeffs_H[0] *= 0
+
+    # reconstruction
+    imArray_H = pywt.waverec2(coeffs_H, mode)
+    imArray_H *= 255
+    imArray_H = np.uint8(imArray_H)
+
+    return imArray_H
+
+def _get_mscn_variance(block, sub_block_size=(50, 50)):
+
+    blocks = segmentation.divide_in_blocks(block, sub_block_size)
+
+    data = []
+
+    for block in blocks:
+        mscn_coefficients = transform.get_mscn_coefficients(block)
+        flat_coeff = mscn_coefficients.flatten()
+        data.append(np.var(flat_coeff))
+
+    return np.sort(data)
+

generateAndTrain_maxwell_custom.sh

@@ -14,7 +14,7 @@ if [ -z "$2" ]
     exit 1
 fi
 
-result_filename="models_info/models_comparisons.csv"
+result_filename="results/models_comparisons.csv"
 VECTOR_SIZE=200
 size=$1
 metric=$2
@@ -55,11 +55,11 @@ for counter in {0..4}; do
 
                     echo "${MODEL_NAME} results already generated..."
                 else
-                    python generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                    python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
                     python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
-                    #python predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
-                    python save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+                    #python prediction/predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
+                    python others/save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
                 fi
             done
         done

generateAndTrain_maxwell_custom_center.sh

@@ -14,7 +14,7 @@ if [ -z "$2" ]
     exit 1
 fi
 
-result_filename="models_info/models_comparisons.csv"
+result_filename="results/models_comparisons.csv"
 VECTOR_SIZE=200
 size=$1
 metric=$2
@@ -55,11 +55,11 @@ for counter in {0..4}; do
 
                     echo "${MODEL_NAME} results already generated..."
                 else
-                    python generate_data_model_random_center.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                    python generate/generate_data_model_random_center.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
                     python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
-                    #python predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
-                    python save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+                    #python prediction/predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
+                    python others/save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
                 fi
             done
         done

generateAndTrain_maxwell_custom_split.sh

@@ -14,7 +14,7 @@ if [ -z "$2" ]
     exit 1
 fi
 
-result_filename="models_info/models_comparisons.csv"
+result_filename="results/models_comparisons.csv"
 VECTOR_SIZE=200
 size=$1
 metric=$2
@@ -55,11 +55,11 @@ for counter in {0..4}; do
 
                     echo "${MODEL_NAME} results already generated..."
                 else
-                    python generate_data_model_random_split.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                    python generate/generate_data_model_random_split.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
                     python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
-                    #python predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
-                    python save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+                    #python prediction/predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
+                    python others/save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
                 fi
             done
         done

generate_all_data.py

@@ -1,27 +1,25 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
 import random
 import time
 import json
 
-from modules.utils.data import get_svd_data
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
-from skimage import color
 
-from modules.utils import config as cfg
+from ipfml.processing import transform, segmentation
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+from data_attributes import get_svd_data
+
 
 # getting configuration information
-config_filename         = cfg.config_filename
 zone_folder             = cfg.zone_folder
 min_max_filename        = cfg.min_max_filename_extension
 
@@ -33,7 +31,7 @@ path                    = cfg.dataset_path
 zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 
-metric_choices          = cfg.metric_choices_labels
+features_choices        = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 
 generic_output_file_svd = '_random.csv'
@@ -41,7 +39,7 @@ generic_output_file_svd = '_random.csv'
 def generate_data_svd(data_type, mode):
     """
     @brief Method which generates all .csv files from scenes
-    @param data_type,  metric choice
+    @param data_type,  feature choice
     @param mode, normalization choice
     @return nothing
     """
@@ -57,20 +55,11 @@ def generate_data_svd(data_type, mode):
     data_min_max_filename = os.path.join(path, data_type + min_max_filename)
 
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         print(folder_scene)
         scene_path = os.path.join(path, folder_scene)
 
-        config_file_path = os.path.join(scene_path, config_filename)
-
-        with open(config_file_path, "r") as config_file:
-            last_image_name = config_file.readline().strip()
-            prefix_image_name = config_file.readline().strip()
-            start_index_image = config_file.readline().strip()
-            end_index_image = config_file.readline().strip()
-            step_counter = int(config_file.readline().strip())
-
         # getting output filename
         output_svd_filename = data_type + "_" + mode + generic_output_file_svd
 
@@ -93,27 +82,21 @@ def generate_data_svd(data_type, mode):
             # add writer into list
             svd_output_files.append(open(svd_file_path, 'w'))
 
-
-        current_counter_index = int(start_index_image)
-        end_counter_index = int(end_index_image)
-
-
-        while(current_counter_index <= end_counter_index):
-
-            current_counter_index_str = str(current_counter_index)
-
-            while len(start_index_image) > len(current_counter_index_str):
-                current_counter_index_str = "0" + current_counter_index_str
-
-            img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+        # get all images of folder
+        scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+        number_scene_image = len(scene_images)
+            
+        for id_img, img_path in enumerate(scene_images):
+            
+            current_image_postfix = dt.get_scene_image_postfix(img_path)
 
             current_img = Image.open(img_path)
-            img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+            img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
             for id_block, block in enumerate(img_blocks):
 
                 ###########################
-                # Metric computation part #
+                # feature computation part #
                 ###########################
 
                 data = get_svd_data(data_type, block)
@@ -151,19 +134,16 @@ def generate_data_svd(data_type, mode):
                 current_file = svd_output_files[id_block]
 
                 # add of index
-                current_file.write(current_counter_index_str + ';')
+                current_file.write(current_image_postfix + ';')
 
                 for val in data:
                     current_file.write(str(val) + ";")
 
                 current_file.write('\n')
 
-            start_index_image_int = int(start_index_image)
-            print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((current_counter_index - start_index_image_int) / (end_counter_index - start_index_image_int)* 100.) + "%")
+            print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((id_img + 1) / number_scene_image * 100.) + "%")
             sys.stdout.write("\033[F")
 
-            current_counter_index += step_counter
-
         for f in svd_output_files:
             f.close()
 
@@ -180,26 +160,26 @@ def generate_data_svd(data_type, mode):
 
 def main():
 
-    parser = argparse.ArgumentParser(description="Compute and prepare data of metric of all scenes (keep in memory min and max value found)")
+    parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes (keep in memory min and max value found)")
 
-    parser.add_argument('--metric', type=str, 
-                                    help="metric choice in order to compute data (use 'all' if all metrics are needed)", 
-                                    choices=metric_choices)
+    parser.add_argument('--feature', type=str, 
+                                    help="feature choice in order to compute data (use 'all' if all features are needed)", 
+                                    choices=features_choices)
 
     args = parser.parse_args()
 
-    p_metric = args.metric
+    p_feature = args.feature
 
-    # generate all or specific metric data
-    if p_metric == 'all':
-        for m in metric_choices:
+    # generate all or specific feature data
+    if p_feature == 'all':
+        for m in features_choices:
             generate_data_svd(m, 'svd')
             generate_data_svd(m, 'svdn')
             generate_data_svd(m, 'svdne')
     else:
-        generate_data_svd(p_metric, 'svd')
-        generate_data_svd(p_metric, 'svdn')
-        generate_data_svd(p_metric, 'svdne')
+        generate_data_svd(p_feature, 'svd')
+        generate_data_svd(p_feature, 'svdn')
+        generate_data_svd(p_feature, 'svdne')
 
 if __name__== "__main__":
     main()

generate_data_model.py

@@ -1,40 +1,36 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
+import pandas as pd
 import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
 
-from modules.utils import config as cfg
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
 from modules.utils import data as dt
+from data_attributes import get_svd_data
+
 
 # getting configuration information
-config_filename         = cfg.config_filename
 learned_folder          = cfg.learned_zones_folder
 min_max_filename        = cfg.min_max_filename_extension
 
-# define all scenes values
+# define all scenes variables
 scenes_list             = cfg.scenes_names
 scenes_indexes          = cfg.scenes_indices
-choices                 = cfg.normalization_choices
 path                    = cfg.dataset_path
 zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 
 renderer_choices        = cfg.renderer_choices
 normalization_choices   = cfg.normalization_choices
-metric_choices          = cfg.metric_choices_labels
+features_choices        = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 custom_min_max_folder   = cfg.min_max_custom_folder
 min_max_ext             = cfg.min_max_filename_extension
@@ -50,15 +46,15 @@ def construct_new_line(path_seuil, interval, line, choice, each, norm):
 
     line_data = line.split(';')
     seuil = line_data[0]
-    metrics = line_data[begin+1:end+1]
+    features = line_data[begin+1:end+1]
 
-    metrics = [float(m) for id, m in enumerate(metrics) if id % each == 0 ]
+    features = [float(m) for id, m in enumerate(features) if id % each == 0 ]
 
     if norm:
         if choice == 'svdne':
-            metrics = utils.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
         if choice == 'svdn':
-            metrics = utils.normalize_arr(metrics)
+            features = utils.normalize_arr(features)
 
     with open(path_seuil, "r") as seuil_file:
         seuil_learned = int(seuil_file.readline().strip())
@@ -68,14 +64,14 @@ def construct_new_line(path_seuil, interval, line, choice, each, norm):
     else:
         line = '0'
 
-    for idx, val in enumerate(metrics):
+    for val in features:
         line += ';'
         line += str(val)
     line += '\n'
 
     return line
 
-def get_min_max_value_interval(_scenes_list, _interval, _metric):
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
 
     global min_value_interval, max_value_interval
 
@@ -84,7 +80,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
 
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         # only take care of maxwell scenes
         if folder_scene in _scenes_list:
@@ -99,9 +95,9 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                     index_str = "0" + index_str
                 zones_folder.append("zone"+index_str)
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
                 zone_path = os.path.join(scene_path, zone_folder)
-                data_filename = _metric + "_svd" + generic_output_file_svd
+                data_filename = _feature + "_svd" + generic_output_file_svd
                 data_file_path = os.path.join(zone_path, data_filename)
 
                 # getting number of line and read randomly lines
@@ -114,11 +110,11 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                     begin, end = _interval
 
                     line_data = line.split(';')
-                    metrics = line_data[begin+1:end+1]
-                    metrics = [float(m) for m in metrics]
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
 
-                    min_value = min(metrics)
-                    max_value = max(metrics)
+                    min_value = min(features)
+                    max_value = max(features)
 
                     if min_value < min_value_interval:
                         min_value_interval = min_value
@@ -127,7 +123,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                         max_value_interval = max_value
 
 
-def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes_list, _zones = zones_indices, _percent = 1, _step=1, _each=1, _norm=False, _custom=False):
+def generate_data_model(_filename, _interval, _choice, _feature, _scenes = scenes_list, _zones = zones_indices, _percent = 1, _step=1, _each=1, _norm=False, _custom=False):
 
     output_train_filename = _filename + ".train"
     output_test_filename = _filename + ".test"
@@ -142,7 +138,7 @@ def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes
     train_file = open(output_train_filename, 'w')
     test_file = open(output_test_filename, 'w')
 
-    for id_scene, folder_scene in enumerate(scenes_list):
+    for folder_scene in scenes_list:
 
         # only take care of maxwell scenes
         scene_path = os.path.join(path, folder_scene)
@@ -172,9 +168,9 @@ def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes
 
             # if custom normalization choices then we use svd values not already normalized
             if _custom:
-                data_filename = _metric + "_svd" + generic_output_file_svd
+                data_filename = _feature + "_svd" + generic_output_file_svd
             else:
-                data_filename = _metric + "_" + _choice + generic_output_file_svd
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
 
             data_file_path = os.path.join(zone_path, data_filename)
 
@@ -220,7 +216,7 @@ def main():
     parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
     parser.add_argument('--zones', type=str, help='Zones indices to use for training data set')
     parser.add_argument('--percent', type=float, help='Percent of data use for train and test dataset (by default 1)', default=1.0)
@@ -234,7 +230,7 @@ def main():
     p_filename = args.output
     p_interval = list(map(int, args.interval.split(',')))
     p_kind     = args.kind
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_scenes   = args.scenes.split(',')
     p_zones    = list(map(int, args.zones.split(',')))
     p_percent  = args.percent
@@ -251,12 +247,12 @@ def main():
     scenes_selected = []
 
     for scene_id in p_scenes:
-        index = scenes_indexes.index(scene_id.strip())
+        index = scenes_indices.index(scene_id.strip())
         scenes_selected.append(scenes_list[index])
 
     # find min max value if necessary to renormalize data
     if p_custom:
-        get_min_max_value_interval(scenes_list, p_interval, p_metric)
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
 
         # write new file to save
         if not os.path.exists(custom_min_max_folder):
@@ -270,7 +266,7 @@ def main():
             f.write(str(max_value_interval) + '\n')
 
     # create database using img folder (generate first time only)
-    generate_data_model(p_filename, p_interval, p_kind, p_metric, scenes_selected, p_zones, p_percent, p_step, p_each, p_custom)
+    generate_data_model(p_filename, p_interval, p_kind, p_feature, scenes_selected, p_zones, p_percent, p_step, p_each, p_custom)
 
 if __name__== "__main__":
     main()

generate_data_model_random.py

@@ -1,30 +1,27 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
+import pandas as pd
 import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
 
-from modules.utils import config as cfg
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
 from modules.utils import data as dt
+from data_attributes import get_svd_data
+
 
 # getting configuration information
-config_filename         = cfg.config_filename
 learned_folder          = cfg.learned_zones_folder
 min_max_filename        = cfg.min_max_filename_extension
 
-# define all scenes values
+# define all scenes variables
 all_scenes_list         = cfg.scenes_names
 all_scenes_indices      = cfg.scenes_indices
 
@@ -34,7 +31,7 @@ zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 
 renderer_choices        = cfg.renderer_choices
-metric_choices          = cfg.metric_choices_labels
+features_choices        = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 custom_min_max_folder   = cfg.min_max_custom_folder
 min_max_ext             = cfg.min_max_filename_extension
@@ -49,18 +46,18 @@ def construct_new_line(path_seuil, interval, line, choice, each, norm):
 
     line_data = line.split(';')
     seuil = line_data[0]
-    metrics = line_data[begin+1:end+1]
+    features = line_data[begin+1:end+1]
 
     # keep only if modulo result is 0 (keep only each wanted values)
-    metrics = [float(m) for id, m in enumerate(metrics) if id % each == 0]
+    features = [float(m) for id, m in enumerate(features) if id % each == 0]
 
     # TODO : check if it's always necessary to do that (loss of information for svd)
     if norm:
 
         if choice == 'svdne':
-            metrics = utils.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
         if choice == 'svdn':
-            metrics = utils.normalize_arr(metrics)
+            features = utils.normalize_arr(features)
 
     with open(path_seuil, "r") as seuil_file:
         seuil_learned = int(seuil_file.readline().strip())
@@ -70,14 +67,14 @@ def construct_new_line(path_seuil, interval, line, choice, each, norm):
     else:
         line = '0'
 
-    for idx, val in enumerate(metrics):
+    for val in features:
         line += ';'
         line += str(val)
     line += '\n'
 
     return line
 
-def get_min_max_value_interval(_scenes_list, _interval, _metric):
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
 
     global min_value_interval, max_value_interval
 
@@ -86,7 +83,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
 
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         # only take care of maxwell scenes
         if folder_scene in _scenes_list:
@@ -101,12 +98,12 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                     index_str = "0" + index_str
                 zones_folder.append("zone"+index_str)
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
 
                 zone_path = os.path.join(scene_path, zone_folder)
 
                 # if custom normalization choices then we use svd values not already normalized
-                data_filename = _metric + "_svd"+ generic_output_file_svd
+                data_filename = _feature + "_svd"+ generic_output_file_svd
 
                 data_file_path = os.path.join(zone_path, data_filename)
 
@@ -121,11 +118,11 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
 
                     line_data = line.split(';')
 
-                    metrics = line_data[begin+1:end+1]
-                    metrics = [float(m) for m in metrics]
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
 
-                    min_value = min(metrics)
-                    max_value = max(metrics)
+                    min_value = min(features)
+                    max_value = max(features)
 
                     if min_value < min_value_interval:
                         min_value_interval = min_value
@@ -134,7 +131,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                         max_value_interval = max_value
 
 
-def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
+def generate_data_model(_scenes_list, _filename, _interval, _choice, _feature, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
 
     output_train_filename = _filename + ".train"
     output_test_filename = _filename + ".test"
@@ -149,7 +146,7 @@ def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _s
     train_file_data = []
     test_file_data  = []
 
-    for id_scene, folder_scene in enumerate(_scenes_list):
+    for folder_scene in _scenes_list:
 
         scene_path = os.path.join(path, folder_scene)
 
@@ -186,9 +183,9 @@ def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _s
 
             # if custom normalization choices then we use svd values not already normalized
             if _custom:
-                data_filename = _metric + "_svd"+ generic_output_file_svd
+                data_filename = _feature + "_svd"+ generic_output_file_svd
             else:
-                data_filename = _metric + "_" + _choice + generic_output_file_svd
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
 
             data_file_path = os.path.join(zone_path, data_filename)
 
@@ -244,7 +241,7 @@ def main():
     parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
     parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set')
     parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
@@ -259,7 +256,7 @@ def main():
     p_filename = args.output
     p_interval = list(map(int, args.interval.split(',')))
     p_kind     = args.kind
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_scenes   = args.scenes.split(',')
     p_nb_zones = args.nb_zones
     p_random   = args.random
@@ -283,7 +280,7 @@ def main():
 
     # find min max value if necessary to renormalize data
     if p_custom:
-        get_min_max_value_interval(scenes_list, p_interval, p_metric)
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
 
         # write new file to save
         if not os.path.exists(custom_min_max_folder):
@@ -297,7 +294,7 @@ def main():
             f.write(str(max_value_interval) + '\n')
 
     # create database using img folder (generate first time only)
-    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
+    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_feature, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
 
 if __name__== "__main__":
     main()

generate_data_model_random_center.py

@@ -1,30 +1,27 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
+import pandas as pd
 import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
 
-from modules.utils import config as cfg
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
 from modules.utils import data as dt
+from data_attributes import get_svd_data
+
 
 # getting configuration information
-config_filename         = cfg.config_filename
 learned_folder          = cfg.learned_zones_folder
 min_max_filename        = cfg.min_max_filename_extension
 
-# define all scenes values
+# define all scenes variables
 all_scenes_list         = cfg.scenes_names
 all_scenes_indices      = cfg.scenes_indices
 
@@ -34,7 +31,7 @@ zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 
 renderer_choices        = cfg.renderer_choices
-metric_choices          = cfg.metric_choices_labels
+features_choices        = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 custom_min_max_folder   = cfg.min_max_custom_folder
 min_max_ext             = cfg.min_max_filename_extension
@@ -51,32 +48,32 @@ def construct_new_line(seuil_learned, interval, line, choice, each, norm):
 
     line_data = line.split(';')
     seuil = line_data[0]
-    metrics = line_data[begin+1:end+1]
+    features = line_data[begin+1:end+1]
 
     # keep only if modulo result is 0 (keep only each wanted values)
-    metrics = [float(m) for id, m in enumerate(metrics) if id % each == 0]
+    features = [float(m) for id, m in enumerate(features) if id % each == 0]
 
     # TODO : check if it's always necessary to do that (loss of information for svd)
     if norm:
 
         if choice == 'svdne':
-            metrics = utils.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
         if choice == 'svdn':
-            metrics = utils.normalize_arr(metrics)
+            features = utils.normalize_arr(features)
 
     if seuil_learned > int(seuil):
         line = '1'
     else:
         line = '0'
 
-    for idx, val in enumerate(metrics):
+    for val in features:
         line += ';'
         line += str(val)
     line += '\n'
 
     return line
 
-def get_min_max_value_interval(_scenes_list, _interval, _metric):
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
 
     global min_value_interval, max_value_interval
 
@@ -85,7 +82,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
 
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         # only take care of maxwell scenes
         if folder_scene in _scenes_list:
@@ -100,12 +97,12 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                     index_str = "0" + index_str
                 zones_folder.append("zone"+index_str)
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
 
                 zone_path = os.path.join(scene_path, zone_folder)
 
                 # if custom normalization choices then we use svd values not already normalized
-                data_filename = _metric + "_svd"+ generic_output_file_svd
+                data_filename = _feature + "_svd"+ generic_output_file_svd
 
                 data_file_path = os.path.join(zone_path, data_filename)
 
@@ -120,11 +117,11 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
 
                     line_data = line.split(';')
 
-                    metrics = line_data[begin+1:end+1]
-                    metrics = [float(m) for m in metrics]
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
 
-                    min_value = min(metrics)
-                    max_value = max(metrics)
+                    min_value = min(features)
+                    max_value = max(features)
 
                     if min_value < min_value_interval:
                         min_value_interval = min_value
@@ -133,7 +130,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                         max_value_interval = max_value
 
 
-def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
+def generate_data_model(_scenes_list, _filename, _interval, _choice, _feature, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
 
     output_train_filename = _filename + ".train"
     output_test_filename = _filename + ".test"
@@ -148,7 +145,7 @@ def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _s
     train_file_data = []
     test_file_data  = []
 
-    for id_scene, folder_scene in enumerate(_scenes_list):
+    for folder_scene in _scenes_list:
 
         scene_path = os.path.join(path, folder_scene)
 
@@ -185,9 +182,9 @@ def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _s
 
             # if custom normalization choices then we use svd values not already normalized
             if _custom:
-                data_filename = _metric + "_svd"+ generic_output_file_svd
+                data_filename = _feature + "_svd"+ generic_output_file_svd
             else:
-                data_filename = _metric + "_" + _choice + generic_output_file_svd
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
 
             data_file_path = os.path.join(zone_path, data_filename)
 
@@ -255,7 +252,7 @@ def main():
     parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
     parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set')
     parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
@@ -270,7 +267,7 @@ def main():
     p_filename = args.output
     p_interval = list(map(int, args.interval.split(',')))
     p_kind     = args.kind
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_scenes   = args.scenes.split(',')
     p_nb_zones = args.nb_zones
     p_random   = args.random
@@ -294,7 +291,7 @@ def main():
 
     # find min max value if necessary to renormalize data
     if p_custom:
-        get_min_max_value_interval(scenes_list, p_interval, p_metric)
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
 
         # write new file to save
         if not os.path.exists(custom_min_max_folder):
@@ -308,7 +305,7 @@ def main():
             f.write(str(max_value_interval) + '\n')
 
     # create database using img folder (generate first time only)
-    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
+    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_feature, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
 
 if __name__== "__main__":
     main()

generate_data_model_random_split.py

@@ -1,30 +1,27 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
+import pandas as pd
 import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
 
-from modules.utils import config as cfg
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
 from modules.utils import data as dt
+from data_attributes import get_svd_data
+
 
 # getting configuration information
-config_filename         = cfg.config_filename
 learned_folder          = cfg.learned_zones_folder
 min_max_filename        = cfg.min_max_filename_extension
 
-# define all scenes values
+# define all scenes variables
 all_scenes_list         = cfg.scenes_names
 all_scenes_indices      = cfg.scenes_indices
 
@@ -34,7 +31,7 @@ zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 
 renderer_choices        = cfg.renderer_choices
-metric_choices          = cfg.metric_choices_labels
+features_choices        = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 custom_min_max_folder   = cfg.min_max_custom_folder
 min_max_ext             = cfg.min_max_filename_extension
@@ -51,32 +48,32 @@ def construct_new_line(seuil_learned, interval, line, choice, each, norm):
 
     line_data = line.split(';')
     seuil = line_data[0]
-    metrics = line_data[begin+1:end+1]
+    features = line_data[begin+1:end+1]
 
     # keep only if modulo result is 0 (keep only each wanted values)
-    metrics = [float(m) for id, m in enumerate(metrics) if id % each == 0]
+    features = [float(m) for id, m in enumerate(features) if id % each == 0]
 
     # TODO : check if it's always necessary to do that (loss of information for svd)
     if norm:
 
         if choice == 'svdne':
-            metrics = utils.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
         if choice == 'svdn':
-            metrics = utils.normalize_arr(metrics)
+            features = utils.normalize_arr(features)
 
     if seuil_learned > int(seuil):
         line = '1'
     else:
         line = '0'
 
-    for idx, val in enumerate(metrics):
+    for val in features:
         line += ';'
         line += str(val)
     line += '\n'
 
     return line
 
-def get_min_max_value_interval(_scenes_list, _interval, _metric):
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
 
     global min_value_interval, max_value_interval
 
@@ -85,7 +82,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
 
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         # only take care of maxwell scenes
         if folder_scene in _scenes_list:
@@ -100,12 +97,12 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                     index_str = "0" + index_str
                 zones_folder.append("zone"+index_str)
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
 
                 zone_path = os.path.join(scene_path, zone_folder)
 
                 # if custom normalization choices then we use svd values not already normalized
-                data_filename = _metric + "_svd"+ generic_output_file_svd
+                data_filename = _feature + "_svd"+ generic_output_file_svd
 
                 data_file_path = os.path.join(zone_path, data_filename)
 
@@ -120,11 +117,11 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
 
                     line_data = line.split(';')
 
-                    metrics = line_data[begin+1:end+1]
-                    metrics = [float(m) for m in metrics]
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
 
-                    min_value = min(metrics)
-                    max_value = max(metrics)
+                    min_value = min(features)
+                    max_value = max(features)
 
                     if min_value < min_value_interval:
                         min_value_interval = min_value
@@ -133,7 +130,7 @@ def get_min_max_value_interval(_scenes_list, _interval, _metric):
                         max_value_interval = max_value
 
 
-def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
+def generate_data_model(_scenes_list, _filename, _interval, _choice, _feature, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
 
     output_train_filename = _filename + ".train"
     output_test_filename = _filename + ".test"
@@ -148,7 +145,7 @@ def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _s
     train_file_data = []
     test_file_data  = []
 
-    for id_scene, folder_scene in enumerate(_scenes_list):
+    for folder_scene in _scenes_list:
 
         scene_path = os.path.join(path, folder_scene)
 
@@ -185,9 +182,9 @@ def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _s
 
             # if custom normalization choices then we use svd values not already normalized
             if _custom:
-                data_filename = _metric + "_svd"+ generic_output_file_svd
+                data_filename = _feature + "_svd"+ generic_output_file_svd
             else:
-                data_filename = _metric + "_" + _choice + generic_output_file_svd
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
 
             data_file_path = os.path.join(zone_path, data_filename)
 
@@ -254,7 +251,7 @@ def main():
     parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
     parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set')
     parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
@@ -269,7 +266,7 @@ def main():
     p_filename = args.output
     p_interval = list(map(int, args.interval.split(',')))
     p_kind     = args.kind
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_scenes   = args.scenes.split(',')
     p_nb_zones = args.nb_zones
     p_random   = args.random
@@ -293,7 +290,7 @@ def main():
 
     # find min max value if necessary to renormalize data
     if p_custom:
-        get_min_max_value_interval(scenes_list, p_interval, p_metric)
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
 
         # write new file to save
         if not os.path.exists(custom_min_max_folder):
@@ -307,7 +304,7 @@ def main():
             f.write(str(max_value_interval) + '\n')
 
     # create database using img folder (generate first time only)
-    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
+    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_feature, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
 
 if __name__== "__main__":
     main()

generate_all_simulate_curves.sh

@@ -1,6 +0,0 @@
-for file in "threshold_map"/*; do
-
-    echo ${file}
-
-    python display_simulation_curves.py --folder ${file}
-done

modules/models.py

@@ -1,3 +1,4 @@
+# models imports
 from sklearn.model_selection import GridSearchCV
 from sklearn.linear_model import LogisticRegression
 from sklearn.ensemble import RandomForestClassifier, VotingClassifier
@@ -70,6 +71,4 @@ def get_trained_model(choice, X_train, y_train):
         return ensemble_model(X_train, y_train)
 
     if choice == 'ensemble_model_v2':
-        return ensemble_model_v2(X_train, y_train)
-
-
+        return ensemble_model_v2(X_train, y_train)

modules/utils/config.py

@@ -1,41 +0,0 @@
-import numpy as np
-
-zone_folder                     = "zone"
-output_data_folder              = 'data'
-dataset_path                    = 'dataset'
-threshold_map_folder            = 'threshold_map'
-models_information_folder       = 'models_info'
-saved_models_folder             = 'saved_models'
-min_max_custom_folder           = 'custom_norm'
-learned_zones_folder            = 'learned_zones'
-correlation_indices_folder      = 'corr_indices'
-
-csv_model_comparisons_filename  = "models_comparisons.csv"
-seuil_expe_filename             = 'seuilExpe'
-min_max_filename_extension      = "_min_max_values"
-config_filename                 = "config"
-
-models_names_list               = ["svm_model","ensemble_model","ensemble_model_v2","deep_keras"]
-
-# define all scenes values
-renderer_choices                = ['all', 'maxwell', 'igloo', 'cycle']
-
-scenes_names                    = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
-scenes_indices                  = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
-
-maxwell_scenes_names            = ['Appart1opt02', 'Cuisine01', 'SdbCentre', 'SdbDroite']
-maxwell_scenes_indices          = ['A', 'D', 'G', 'H']
-
-igloo_scenes_names              = ['Bureau1', 'PNDVuePlongeante']
-igloo_scenes_indices            = ['B', 'F']
-
-cycle_scenes_names              = ['EchecBas', 'Selles']
-cycle_scenes_indices            = ['E', 'I']
-
-normalization_choices           = ['svd', 'svdn', 'svdne']
-zones_indices                   = np.arange(16)
-
-metric_choices_labels           = ['filters_statistics']
-
-keras_epochs                    = 100
-keras_batch                     = 32

modules/utils/data.py

@@ -1,137 +0,0 @@
-from ipfml import processing, metrics, utils
-
-from modules.utils.config import *
-from modules.utils.filters import w2d
-
-import cv2
-
-from PIL import Image
-from skimage import color
-from sklearn.decomposition import FastICA
-from sklearn.decomposition import IncrementalPCA
-from sklearn.decomposition import TruncatedSVD
-from numpy.linalg import svd as lin_svd
-
-from scipy.signal import medfilt2d, wiener, cwt
-
-import numpy as np
-
-
-_scenes_names_prefix   = '_scenes_names'
-_scenes_indices_prefix = '_scenes_indices'
-
-# store all variables from current module context
-context_vars = vars()
-
-
-def get_svd_data(data_type, block):
-    """
-    Method which returns the data type expected
-    """
-
-    if 'filters_statistics' in data_type:
-
-        img_width, img_height = 200, 200
-
-        lab_img = metrics.get_LAB_L(block)
-        arr = np.array(lab_img)
-
-        # compute all filters statistics
-        def get_stats(arr, I_filter):
-
-            e1       = np.abs(arr - I_filter)
-            L        = np.array(e1)
-            mu0      = np.mean(L)
-            A        = L - mu0
-            H        = A * A
-            E        = np.sum(H) / (img_width * img_height)
-            P        = np.sqrt(E)
-
-            return mu0, P
-
-        stats = []
-
-        kernel = np.ones((3,3),np.float32)/9
-        stats.append(get_stats(arr, cv2.filter2D(arr,-1,kernel)))
-
-        kernel = np.ones((5,5),np.float32)/25
-        stats.append(get_stats(arr, cv2.filter2D(arr,-1,kernel)))
-
-        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (3, 3), 0.5)))
-
-        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (3, 3), 1)))
-
-        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (3, 3), 1.5)))
-
-        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (5, 5), 0.5)))
-
-        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (5, 5), 1)))
-
-        stats.append(get_stats(arr, cv2.GaussianBlur(arr, (5, 5), 1.5)))
-
-        stats.append(get_stats(arr, medfilt2d(arr, [3, 3])))
-
-        stats.append(get_stats(arr, medfilt2d(arr, [5, 5])))
-
-        stats.append(get_stats(arr, wiener(arr, [3, 3])))
-
-        stats.append(get_stats(arr, wiener(arr, [5, 5])))
-
-        wave = w2d(arr, 'db1', 2)
-        stats.append(get_stats(arr, np.array(wave, 'float64')))
-
-        data = []
-
-        for stat in stats:
-            data.append(stat[0])
-
-        for stat in stats:
-            data.append(stat[1])
-        
-        data = np.array(data)
-
-    return data
-
-
-def get_highest_values(arr, n):
-    return np.array(arr).argsort()[-n:][::-1]
-
-
-def get_lowest_values(arr, n):
-    return np.array(arr).argsort()[::-1][-n:][::-1]
-
-
-def _get_mscn_variance(block, sub_block_size=(50, 50)):
-
-    blocks = processing.divide_in_blocks(block, sub_block_size)
-
-    data = []
-
-    for block in blocks:
-        mscn_coefficients = processing.get_mscn_coefficients(block)
-        flat_coeff = mscn_coefficients.flatten()
-        data.append(np.var(flat_coeff))
-
-    return np.sort(data)
-
-
-def get_renderer_scenes_indices(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_indices
-    else:
-        return context_vars[renderer_name + _scenes_indices_prefix]
-
-def get_renderer_scenes_names(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_names
-    else:
-        return context_vars[renderer_name + _scenes_names_prefix]
-

modules/utils/filters.py

@@ -1,22 +0,0 @@
-import cv2, pywt
-import numpy as np
-from scipy.signal import medfilt2d, wiener, cwt
-
-def w2d(arr, mode='haar', level=1):
-    #convert to float   
-    imArray = arr
-    imArray /= 255
-
-    # compute coefficients 
-    coeffs=pywt.wavedec2(imArray, mode, level=level)
-
-    #Process Coefficients
-    coeffs_H=list(coeffs)  
-    coeffs_H[0] *= 0
-
-    # reconstruction
-    imArray_H = pywt.waverec2(coeffs_H, mode);
-    imArray_H *= 255
-    imArray_H = np.uint8(imArray_H)
-
-    return imArray_H

predict_noisy_image_svd.py

@@ -1,20 +1,25 @@
-from sklearn.externals import joblib
-
+# main imports
+import sys, os, argparse, json
 import numpy as np
 
+# models imports
+from keras.models import model_from_json
+from sklearn.externals import joblib
+
+# image processing imports
 from ipfml import processing, utils
 from PIL import Image
 
-import sys, os, argparse, json
-
-from keras.models import model_from_json
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-from modules.utils import config as cfg
-from modules.utils import data as dt
+import custom_config as cfg
+from data_attributes import get_svd_data
 
+# variables and parameters
 path                  = cfg.dataset_path
 min_max_ext           = cfg.min_max_filename_extension
-metric_choices        = cfg.metric_choices_labels
+features_choices      = cfg.features_choices_labels
 normalization_choices = cfg.normalization_choices
 
 custom_min_max_folder = cfg.min_max_custom_folder
@@ -28,7 +33,7 @@ def main():
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
 
     args = parser.parse_args()
@@ -37,7 +42,7 @@ def main():
     p_model_file = args.model
     p_interval   = list(map(int, args.interval.split(',')))
     p_mode       = args.mode
-    p_metric     = args.metric
+    p_feature    = args.feature
     p_custom     = args.custom
 
     if '.joblib' in p_model_file:
@@ -69,12 +74,12 @@ def main():
 
             model.compile(loss='binary_crossentropy',
                         optimizer='adam',
-                        metrics=['accuracy'])
+                        features=['accuracy'])
 
     # load image
     img = Image.open(p_img_file)
 
-    data = dt.get_svd_data(p_metric, img)
+    data = get_svd_data(p_feature, img)
 
     # get interval values
     begin, end = p_interval
@@ -109,7 +114,7 @@ def main():
         if p_mode == 'svdne':
 
             # set min_max_filename if custom use
-            min_max_file_path = path + '/' + p_metric + min_max_ext
+            min_max_file_path = path + '/' + p_feature + min_max_ext
 
             # need to read min_max_file
             file_path = os.path.join(os.path.dirname(__file__), min_max_file_path)

predict_seuil_expe.py

@@ -1,17 +1,24 @@
-from sklearn.externals import joblib
-
+# main imports
+import sys, os, argparse
+import subprocess
+import time
 import numpy as np
 
-from ipfml import processing, utils
+# image processing imports
+from ipfml.processing import segmentation
 from PIL import Image
 
-import sys, os, argparse
-import subprocess
-import time
+# models imports
+from sklearn.externals import joblib
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-from modules.utils import config as cfg
+import custom_config as cfg
+from modules.utils import data as dt
 
-config_filename           = cfg.config_filename
+
+# variables and parameters
 scenes_path               = cfg.dataset_path
 min_max_filename          = cfg.min_max_filename_extension
 threshold_expe_filename   = cfg.seuil_expe_filename
@@ -21,7 +28,7 @@ threshold_map_file_prefix = cfg.threshold_map_folder + "_"
 
 zones                     = cfg.zones_indices
 normalization_choices     = cfg.normalization_choices
-metric_choices            = cfg.metric_choices_labels
+features_choices          = cfg.features_choices_labels
 
 tmp_filename              = '/tmp/__model__img_to_predict.png'
 
@@ -36,8 +43,8 @@ def main():
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
-    #parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
+    parser.add_argument('--feature', type=str, help='Feature data choice', choices=features_choices)
+    parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
     parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
 
     args = parser.parse_args()
@@ -45,8 +52,8 @@ def main():
     p_interval   = list(map(int, args.interval.split(',')))
     p_model_file = args.model
     p_mode       = args.mode
-    p_metric     = args.metric
-    #p_limit      = args.limit
+    p_feature     = args.feature
+    p_limit      = args.limit
     p_custom     = args.custom
 
     scenes = os.listdir(scenes_path)
@@ -59,20 +66,17 @@ def main():
 
         scene_path = os.path.join(scenes_path, folder_scene)
 
-        config_path = os.path.join(scene_path, config_filename)
-
-        with open(config_path, "r") as config_file:
-            last_image_name = config_file.readline().strip()
-            prefix_image_name = config_file.readline().strip()
-            start_index_image = config_file.readline().strip()
-            end_index_image = config_file.readline().strip()
-            step_counter = int(config_file.readline().strip())
-
         threshold_expes = []
         threshold_expes_detected = []
         threshold_expes_counter = []
         threshold_expes_found = []
 
+            # get all images of folder
+        scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+        start_quality_image = dt.get_scene_image_quality(scene_images[0])
+        end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
+       
         # get zones list info
         for index in zones:
             index_str = str(index)
@@ -89,29 +93,26 @@ def main():
                 # Initialize default data to get detected model threshold found
                 threshold_expes_detected.append(False)
                 threshold_expes_counter.append(0)
-                threshold_expes_found.append(int(end_index_image)) # by default use max
-
-        current_counter_index = int(start_index_image)
-        end_counter_index = int(end_index_image)
+                threshold_expes_found.append(end_quality_image) # by default use max
 
-        print(current_counter_index)
         check_all_done = False
 
-        while(current_counter_index <= end_counter_index and not check_all_done):
-
-            current_counter_index_str = str(current_counter_index)
-
-            while len(start_index_image) > len(current_counter_index_str):
-                current_counter_index_str = "0" + current_counter_index_str
-
-            img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+        # for each images
+        for img_path in scene_images:
 
             current_img = Image.open(img_path)
-            img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+            current_quality_image = dt.get_scene_image_quality(img_path)
+            current_image_potfix = dt.get_scene_image_postfix(img_path)
 
+            img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
+            current_img = Image.open(img_path)
+            img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
             check_all_done = all(d == True for d in threshold_expes_detected)
 
+            if check_all_done:
+                break
+
             for id_block, block in enumerate(img_blocks):
 
                 # check only if necessary for this scene (not already detected)
@@ -120,11 +121,11 @@ def main():
                     tmp_file_path = tmp_filename.replace('__model__',  p_model_file.split('/')[-1].replace('.joblib', '_'))
                     block.save(tmp_file_path)
 
-                    python_cmd = "python predict_noisy_image_svd.py --image " + tmp_file_path + \
+                    python_cmd = "python prediction/predict_noisy_image_svd.py --image " + tmp_file_path + \
                                     " --interval '" + p_interval + \
                                     "' --model " + p_model_file  + \
                                     " --mode " + p_mode + \
-                                    " --metric " + p_metric
+                                    " --feature " + p_feature
 
                     # specify use of custom file for min max normalization
                     if p_custom:
@@ -148,11 +149,10 @@ def main():
 
                     if threshold_expes_counter[id_block] == p_limit:
                         threshold_expes_detected[id_block] = True
-                        threshold_expes_found[id_block] = current_counter_index
+                        threshold_expes_found[id_block] = current_quality_image
 
-                    print(str(id_block) + " : " + str(current_counter_index) + "/" + str(threshold_expes[id_block]) + " => " + str(prediction))
+                    print(str(id_block) + " : " + current_image_potfix + "/" + str(threshold_expes[id_block]) + " => " + str(prediction))
 
-            current_counter_index += step_counter
             print("------------------------")
             print("Scene " + str(id_scene + 1) + "/" + str(len(scenes)))
             print("------------------------")
@@ -192,12 +192,12 @@ def main():
         avg_abs_dist = sum(abs_dist) / len(abs_dist)
 
         f_map.write('\nScene information : ')
-        f_map.write('\n- BEGIN : ' + str(start_index_image))
-        f_map.write('\n- END : ' + str(end_index_image))
+        f_map.write('\n- BEGIN : ' + str(start_quality_image))
+        f_map.write('\n- END : ' + str(end_quality_image))
 
         f_map.write('\n\nDistances information : ')
         f_map.write('\n- MIN : ' + str(min_abs_dist))
-        f_map.write('\n- MAX : ' + str(max_abs_dist))
+        f_map.write('\n- MAX : ' + str(max_abs_dist))          
         f_map.write('\n- AVG : ' + str(avg_abs_dist))
 
         f_map.write('\n\nOther information : ')
@@ -209,7 +209,7 @@ def main():
         print("Scene " + str(id_scene + 1) + "/" + str(len(scenes)) + " Done..")
         print("------------------------")
 
-        time.sleep(10)
+        time.sleep(1)
 
 
 if __name__== "__main__":

predict_seuil_expe_maxwell.py

@@ -1,18 +1,24 @@
-from sklearn.externals import joblib
-
+# main imports
+import sys, os, argparse
+import subprocess
+import time
 import numpy as np
 
-from ipfml import processing
+# image processing imports
+from ipfml.processing import segmentation
 from PIL import Image
 
-import sys, os, argparse
-import subprocess
-import time
+# models imports
+from sklearn.externals import joblib
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
+import custom_config as cfg
+from modules.utils import data as dt
 
-from modules.utils import config as cfg
 
-config_filename           = cfg.config_filename
+# variables and parameters
 scenes_path               = cfg.dataset_path
 min_max_filename          = cfg.min_max_filename_extension
 threshold_expe_filename   = cfg.seuil_expe_filename
@@ -23,7 +29,7 @@ threshold_map_file_prefix = cfg.threshold_map_folder + "_"
 zones                     = cfg.zones_indices
 maxwell_scenes            = cfg.maxwell_scenes_names
 normalization_choices     = cfg.normalization_choices
-metric_choices            = cfg.metric_choices_labels
+features_choices          = cfg.features_choices_labels
 
 tmp_filename              = '/tmp/__model__img_to_predict.png'
 
@@ -39,8 +45,8 @@ def main():
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
-    #parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
+    parser.add_argument('--feature', type=str, help='Feature data choice', choices=features_choices)
+    parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
     parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
 
     args = parser.parse_args()
@@ -48,8 +54,8 @@ def main():
     p_interval   = list(map(int, args.interval.split(',')))
     p_model_file = args.model
     p_mode       = args.mode
-    p_metric     = args.metric
-    #p_limit      = args.limit
+    p_feature    = args.feature
+    p_limit      = args.limit
     p_custom     = args.custom
 
     scenes = os.listdir(scenes_path)
@@ -65,20 +71,18 @@ def main():
 
             scene_path = os.path.join(scenes_path, folder_scene)
 
-            config_path = os.path.join(scene_path, config_filename)
-
-            with open(config_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             threshold_expes = []
             threshold_expes_detected = []
             threshold_expes_counter = []
             threshold_expes_found = []
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
+    
+
             # get zones list info
             for index in zones:
                 index_str = str(index)
@@ -95,29 +99,23 @@ def main():
                     # Initialize default data to get detected model threshold found
                     threshold_expes_detected.append(False)
                     threshold_expes_counter.append(0)
-                    threshold_expes_found.append(int(end_index_image)) # by default use max
-
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
+                    threshold_expes_found.append(end_quality_image) # by default use max
 
-            print(current_counter_index)
             check_all_done = False
 
-            while(current_counter_index <= end_counter_index and not check_all_done):
-
-                current_counter_index_str = str(current_counter_index)
-
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
-
-                img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+            # for each images
+            for img_path in scene_images:
 
                 current_img = Image.open(img_path)
-                img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+                current_postfix_image = dt.get_scene_image_postfix(img_path)
 
+                img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
                 check_all_done = all(d == True for d in threshold_expes_detected)
 
+                if check_all_done:
+                    break
+
                 for id_block, block in enumerate(img_blocks):
 
                     # check only if necessary for this scene (not already detected)
@@ -126,11 +124,11 @@ def main():
                         tmp_file_path = tmp_filename.replace('__model__',  p_model_file.split('/')[-1].replace('.joblib', '_'))
                         block.save(tmp_file_path)
 
-                        python_cmd = "python predict_noisy_image_svd.py --image " + tmp_file_path + \
+                        python_cmd = "python prediction/predict_noisy_image_svd.py --image " + tmp_file_path + \
                                         " --interval '" + p_interval + \
                                         "' --model " + p_model_file  + \
                                         " --mode " + p_mode + \
-                                        " --metric " + p_metric
+                                        " --feature " + p_feature
 
                         # specify use of custom file for min max normalization
                         if p_custom:
@@ -153,11 +151,10 @@ def main():
 
                         if threshold_expes_counter[id_block] == p_limit:
                             threshold_expes_detected[id_block] = True
-                            threshold_expes_found[id_block] = current_counter_index
+                            threshold_expes_found[id_block] = int(current_postfix_image)
 
-                        print(str(id_block) + " : " + str(current_counter_index) + "/" + str(threshold_expes[id_block]) + " => " + str(prediction))
+                        print(str(id_block) + " : " + current_postfix_image + "/" + str(threshold_expes[id_block]) + " => " + str(prediction))
 
-                current_counter_index += step_counter
                 print("------------------------")
                 print("Scene " + str(id_scene + 1) + "/" + str(len(maxwell_scenes)))
                 print("------------------------")
@@ -197,8 +194,8 @@ def main():
             avg_abs_dist = sum(abs_dist) / len(abs_dist)
 
             f_map.write('\nScene information : ')
-            f_map.write('\n- BEGIN : ' + str(start_index_image))
-            f_map.write('\n- END : ' + str(end_index_image))
+            f_map.write('\n- BEGIN : ' + str(start_quality_image))
+            f_map.write('\n- END : ' + str(end_quality_image))
 
             f_map.write('\n\nDistances information : ')
             f_map.write('\n- MIN : ' + str(min_abs_dist))

predict_seuil_expe_maxwell_curve.py

@@ -1,17 +1,24 @@
-from sklearn.externals import joblib
-
+# main imports
+import sys, os, argparse
+import subprocess
+import time
 import numpy as np
 
-from ipfml import processing
+# image processing imports
+from ipfml.processing import segmentation
 from PIL import Image
 
-import sys, os, argparse
-import subprocess
-import time
+# models imports
+from sklearn.externals import joblib
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-from modules.utils import config as cfg
+import custom_config as cfg
+from modules.utils import data as dt
 
-config_filename           = cfg.config_filename
+
+# variables and parameters
 scenes_path               = cfg.dataset_path
 min_max_filename          = cfg.min_max_filename_extension
 threshold_expe_filename   = cfg.seuil_expe_filename
@@ -22,7 +29,7 @@ threshold_map_file_prefix = cfg.threshold_map_folder + "_"
 zones                     = cfg.zones_indices
 maxwell_scenes            = cfg.maxwell_scenes_names
 normalization_choices     = cfg.normalization_choices
-metric_choices            = cfg.metric_choices_labels
+features_choices          = cfg.features_choices_labels
 
 simulation_curves_zones   = "simulation_curves_zones_"
 tmp_filename              = '/tmp/__model__img_to_predict.png'
@@ -39,7 +46,7 @@ def main():
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=normalization_choices)
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     #parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
     parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
 
@@ -49,7 +56,7 @@ def main():
     p_interval   = args.interval
     p_model_file = args.model
     p_mode       = args.mode
-    p_metric     = args.metric
+    p_feature    = args.feature
     #p_limit      = args.limit
     p_custom     = args.custom
 
@@ -68,19 +75,18 @@ def main():
 
             scene_path = os.path.join(scenes_path, folder_scene)
 
-            config_path = os.path.join(scene_path, config_filename)
-
-            with open(config_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             threshold_expes = []
             threshold_expes_found = []
             block_predictions_str = []
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
+            # using first two images find the step of quality used
+            quality_step_image  = dt.get_scene_image_quality(scene_images[1]) - start_quality_image
+
             # get zones list info
             for index in zones:
                 index_str = str(index)
@@ -95,26 +101,18 @@ def main():
                     threshold_expes.append(threshold)
 
                     # Initialize default data to get detected model threshold found
-                    threshold_expes_found.append(int(end_index_image)) # by default use max
+                    threshold_expes_found.append(end_quality_image) # by default use max
 
-                block_predictions_str.append(index_str + ";" + p_model_file + ";" + str(threshold) + ";" + str(start_index_image) + ";" + str(step_counter))
+                block_predictions_str.append(index_str + ";" + p_model_file + ";" + str(threshold) + ";" + str(start_quality_image) + ";" + str(quality_step_image))
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
 
-            print(current_counter_index)
-
-            while(current_counter_index <= end_counter_index):
-
-                current_counter_index_str = str(current_counter_index)
-
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
-
-                img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+            # for each images
+            for img_path in scene_images:
 
                 current_img = Image.open(img_path)
-                img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+                current_quality_image = dt.get_scene_image_quality(img_path)
+
+                img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
                 for id_block, block in enumerate(img_blocks):
 
@@ -124,8 +122,8 @@ def main():
                         tmp_file_path = tmp_filename.replace('__model__',  p_model_file.split('/')[-1].replace('.joblib', '_'))
                         block.save(tmp_file_path)
 
-                        python_cmd_line = "python predict_noisy_image_svd.py --image {0} --interval '{1}' --model {2} --mode {3} --metric {4}"
-                        python_cmd = python_cmd_line.format(tmp_file_path, p_interval, p_model_file, p_mode, p_metric) 
+                        python_cmd_line = "python prediction/predict_noisy_image_svd.py --image {0} --interval '{1}' --model {2} --mode {3} --feature {4}"
+                        python_cmd = python_cmd_line.format(tmp_file_path, p_interval, p_model_file, p_mode, p_feature) 
 
                         # specify use of custom file for min max normalization
                         if p_custom:
@@ -144,9 +142,8 @@ def main():
                         # save here in specific file of block all the predictions done
                         block_predictions_str[id_block] = block_predictions_str[id_block] + ";" + str(prediction)
 
-                        print(str(id_block) + " : " + str(current_counter_index) + "/" + str(threshold_expes[id_block]) + " => " + str(prediction))
+                        print(str(id_block) + " : " + str(current_quality_image) + "/" + str(threshold_expes[id_block]) + " => " + str(prediction))
 
-                current_counter_index += step_counter
                 print("------------------------")
                 print("Scene " + str(id_scene + 1) + "/" + str(len(scenes)))
                 print("------------------------")
@@ -175,4 +172,4 @@ def main():
 
 
 if __name__== "__main__":
-    main()
+    main()

prediction_scene.py

@@ -1,8 +1,11 @@
-from sklearn.externals import joblib
-
+# main imports
+import sys, os, argparse
 import numpy as np
-
+import json
 import pandas as pd
+
+# models imports
+from sklearn.externals import joblib
 from sklearn.metrics import accuracy_score
 from keras.models import Sequential
 from keras.layers import Conv1D, MaxPooling1D
@@ -11,11 +14,12 @@ from keras import backend as K
 from keras.models import model_from_json
 from keras.wrappers.scikit_learn import KerasClassifier
 
-import sys, os, argparse
-import json
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-from modules.utils import config as cfg
+import custom_config as cfg
 
+# parameters and variables
 output_model_folder = cfg.saved_models_folder
 
 def main():

requirements.txt

@@ -8,4 +8,5 @@ Pillow
 pydot
 matplotlib
 path.py
-pandas
+pandas
+opencv-python

runAll_maxwell.sh

@@ -1,7 +1,7 @@
 #! bin/bash
 
-# erase "models_info/models_comparisons.csv" file and write new header
-file_path='models_info/models_comparisons.csv'
+# erase "results/models_comparisons.csv" file and write new header
+file_path='results/models_comparisons.csv'
 
 erased=$1
 
@@ -19,6 +19,6 @@ fi
 for size in {"4","8","16","26","32","40"}; do
 
     for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2"}; do
-        bash generateAndTrain_maxwell.sh ${size} ${metric}
+        bash data_processing/generateAndTrain_maxwell.sh ${size} ${metric}
     done
 done

runAll_maxwell_custom.sh

@@ -1,7 +1,7 @@
 #! bin/bash
 
-# erase "models_info/models_comparisons.csv" file and write new header
-file_path='models_info/models_comparisons.csv'
+# erase "results/models_comparisons.csv" file and write new header
+file_path='results/models_comparisons.csv'
 
 erased=$1
 
@@ -19,6 +19,6 @@ fi
 for size in {"4","8","16","26","32","40"}; do
 
     for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
-        bash generateAndTrain_maxwell_custom.sh ${size} ${metric}
+        bash data_processing/generateAndTrain_maxwell_custom.sh ${size} ${metric}
     done
 done

runAll_maxwell_custom_center.sh

@@ -1,7 +1,7 @@
 #! bin/bash
 
-# erase "models_info/models_comparisons.csv" file and write new header
-file_path='models_info/models_comparisons.csv'
+# erase "results/models_comparisons.csv" file and write new header
+file_path='results/models_comparisons.csv'
 
 erased=$1
 
@@ -19,6 +19,6 @@ fi
 for size in {"4","8","16","26","32","40"}; do
 
     for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
-        bash generateAndTrain_maxwell_custom_center.sh ${size} ${metric}
+        bash data_processing/generateAndTrain_maxwell_custom_center.sh ${size} ${metric}
     done
 done

runAll_maxwell_custom_split.sh

@@ -1,7 +1,7 @@
 #! bin/bash
 
-# erase "models_info/models_comparisons.csv" file and write new header
-file_path='models_info/models_comparisons.csv'
+# erase "results/models_comparisons.csv" file and write new header
+file_path='results/models_comparisons.csv'
 
 erased=$1
 
@@ -19,6 +19,6 @@ fi
 for size in {"4","8","16","26","32","40"}; do
 
     for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
-        bash generateAndTrain_maxwell_custom_split.sh ${size} ${metric}
+        bash data_processing/generateAndTrain_maxwell_custom_split.sh ${size} ${metric}
     done
 done

simulation/generate_all_simulate_curves.sh

@@ -0,0 +1,6 @@
+for file in "threshold_map"/*; do
+
+    echo ${file}
+
+    python display/display_simulation_curves.py --folder ${file}
+done

run_maxwell_simulation.sh

@@ -38,13 +38,13 @@ for size in {"4","8","16","26","32","40"}; do
                             echo "Run simulation for model ${MODEL_NAME}"
 
                             # by default regenerate model
-                            python generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1
+                            python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1
 
                             python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
-                            python predict_seuil_expe_maxwell_curve.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2'
+                            python prediction/predict_seuil_expe_maxwell_curve.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2'
 
-                            python save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+                            python others/save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
 
                         fi
                     done

run_maxwell_simulation_custom.sh

@@ -39,13 +39,13 @@ for size in {"4","8","16","26","32","40"}; do
                             echo "Run simulation for model ${MODEL_NAME}"
 
                             # by default regenerate model
-                            python generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                            python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
 
                             python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
-                            python predict_seuil_expe_maxwell_curve.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
+                            python prediction/predict_seuil_expe_maxwell_curve.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
 
-                            python save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+                            python others/save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
 
                         fi
                     done

run_maxwell_simulation_filters_statistics.sh

@@ -27,12 +27,12 @@ for nb_zones in {4,6,8,10,12}; do
                 echo "${MODEL_NAME} results already generated..."
             else
                 # Use of already generated model
-                # python generate_data_model_random.py --output ${FILENAME} --interval "0,${size}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                # python generate/generate_data_model_random.py --output ${FILENAME} --interval "0,${size}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
                 # python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
-                python predict_seuil_expe_maxwell_curve.py --interval "0,${size}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --custom ${CUSTOM_MIN_MAX_FILENAME}
+                python prediction/predict_seuil_expe_maxwell_curve.py --interval "0,${size}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --custom ${CUSTOM_MIN_MAX_FILENAME}
 
-                python save_model_result_in_md_maxwell.py --interval "0,${size}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+                python others/save_model_result_in_md_maxwell.py --interval "0,${size}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
             fi
         done
     done

train_model.py

@@ -1,3 +1,9 @@
+# main imports
+import numpy as np
+import pandas as pd
+import sys, os, argparse
+
+# models imports
 from sklearn.model_selection import train_test_split
 from sklearn.model_selection import GridSearchCV
 from sklearn.linear_model import LogisticRegression
@@ -9,17 +15,17 @@ from sklearn.externals import joblib
 from sklearn.metrics import accuracy_score, f1_score
 from sklearn.model_selection import cross_val_score
 
-import numpy as np
-import pandas as pd
-import sys, os, argparse
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-from modules.utils import config as cfg
-from modules import models as mdl
+import custom_config as cfg
+import models as mdl
 
+# variables and parameters
 saved_models_folder = cfg.saved_models_folder
 models_list         = cfg.models_names_list
 
-current_dirpath = os.getcwd()
+current_dirpath     = os.getcwd()
 output_model_folder = os.path.join(current_dirpath, saved_models_folder)