Refactoring of data generation scripts

data_processing/generateAndTrain_maxwell.sh

@@ -14,10 +14,19 @@ if [ -z "$2" ]
     exit 1
 fi
 
+if [ -z "$3" ]
+  then
+    echo "No argument supplied"
+    echo "Need of kind of data to use"
+    exit 1
+fi
+
 result_filename="results/models_comparisons.csv"
 VECTOR_SIZE=200
+
 size=$1
 feature=$2
+data=$3
 
 # selection of four scenes (only maxwell)
 scenes="A, D, G, H"
@@ -54,7 +63,7 @@ for counter in {0..4}; do
 
                     echo "${MODEL_NAME} results already generated..."
                 else
-                    python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --renderer "maxwell" --step 40 --random 1 --percent 1
+                    python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --renderer "maxwell" --step 40 --random 1 --percent 1
                     python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                     #python prediction/predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature} --limit_detection '2'

data_processing/generateAndTrain_maxwell_custom.sh

@@ -14,10 +14,19 @@ if [ -z "$2" ]
     exit 1
 fi
 
+if [ -z "$3" ]
+  then
+    echo "No argument supplied"
+    echo "Need of kind of data to use"
+    exit 1
+fi
+
 result_filename="results/models_comparisons.csv"
 VECTOR_SIZE=200
+
 size=$1
 feature=$2
+data=$3
 
 # selection of four scenes (only maxwell)
 scenes="A, D, G, H"
@@ -55,7 +64,7 @@ for counter in {0..4}; do
 
                     echo "${MODEL_NAME} results already generated..."
                 else
-                    python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                    python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
                     python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                     #python prediction/predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}

data_processing/generateAndTrain_maxwell_custom_center.sh

@@ -1,74 +0,0 @@
-#! bin/bash
-
-if [ -z "$1" ]
-  then
-    echo "No argument supplied"
-    echo "Need of vector size"
-    exit 1
-fi
-
-if [ -z "$2" ]
-  then
-    echo "No argument supplied"
-    echo "Need of feature information"
-    exit 1
-fi
-
-result_filename="results/models_comparisons.csv"
-VECTOR_SIZE=200
-size=$1
-feature=$2
-
-# selection of four scenes (only maxwell)
-scenes="A, D, G, H"
-
-half=$(($size/2))
-start=-$half
-for counter in {0..4}; do
-    end=$(($start+$size))
-
-    if [ "$end" -gt "$VECTOR_SIZE" ]; then
-        start=$(($VECTOR_SIZE-$size))
-        end=$(($VECTOR_SIZE))
-    fi
-
-    if [ "$start" -lt "0" ]; then
-        start=$((0))
-        end=$(($size))
-    fi
-
-    for nb_zones in {4,6,8,10,12}; do
-
-        echo $start $end
-
-        for mode in {"svd","svdn","svdne"}; do
-            for model in {"svm_model","ensemble_model","ensemble_model_v2"}; do
-
-                FILENAME="data/${model}_N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${feature}_${mode}"
-                MODEL_NAME="${model}_N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${feature}_${mode}"
-                CUSTOM_MIN_MAX_FILENAME="N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${feature}_${mode}_min_max"
-
-                echo $FILENAME
-
-                # only compute if necessary (perhaps server will fall.. Just in case)
-                if grep -q "${MODEL_NAME}" "${result_filename}"; then
-
-                    echo "${MODEL_NAME} results already generated..."
-                else
-                    python generate/generate_data_model_random_center.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
-                    python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
-
-                    #python prediction/predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
-                    python others/save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}
-                fi
-            done
-        done
-    done
-
-    if [ "$counter" -eq "0" ]; then
-        start=$(($start+50-$half))
-    else
-        start=$(($start+50))
-    fi
-
-done

data_processing/generateAndTrain_maxwell_custom_filters.sh

@@ -14,10 +14,19 @@ if [ -z "$2" ]
     exit 1
 fi
 
+if [ -z "$3" ]
+  then
+    echo "No argument supplied"
+    echo "Need of kind of data to use"
+    exit 1
+fi
+
 result_filename="results/models_comparisons.csv"
 VECTOR_SIZE=200
+
 size=$1
 feature=$2
+data=$3
 
 # selection of four scenes (only maxwell)
 scenes="A, D, G, H"
@@ -37,7 +46,7 @@ for nb_zones in {4,6,8,10,12}; do
 
                 echo "${MODEL_NAME} results already generated..."
             else
-                python generate/generate_data_model_random.py --output ${FILENAME} --interval "0,${size}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "0,${size}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
                 python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                 python others/save_model_result_in_md_maxwell.py --interval "0,${size}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}

data_processing/generateAndTrain_maxwell_custom_filters_center.sh

@@ -1,48 +0,0 @@
-#! bin/bash
-
-if [ -z "$1" ]
-  then
-    echo "No argument supplied"
-    echo "Need of vector size"
-    exit 1
-fi
-
-if [ -z "$2" ]
-  then
-    echo "No argument supplied"
-    echo "Need of feature information"
-    exit 1
-fi
-
-result_filename="results/models_comparisons.csv"
-VECTOR_SIZE=200
-size=$1
-feature=$2
-
-# selection of four scenes (only maxwell)
-scenes="A, D, G, H"
-
-for nb_zones in {4,6,8,10,12}; do
-    for mode in {"svd","svdn","svdne"}; do
-        for model in {"svm_model","ensemble_model","ensemble_model_v2"}; do
-
-            FILENAME="data/${model}_N${size}_B0_E${size}_nb_zones_${nb_zones}_${feature}_${mode}"
-            MODEL_NAME="${model}_N${size}_B0_E${size}_nb_zones_${nb_zones}_${feature}_${mode}"
-            CUSTOM_MIN_MAX_FILENAME="N${size}_B0_E${size}_nb_zones_${nb_zones}_${feature}_${mode}_min_max"
-
-            echo $FILENAME
-
-            # only compute if necessary (perhaps server will fall.. Just in case)
-            if grep -q "${MODEL_NAME}" "${result_filename}"; then
-
-                echo "${MODEL_NAME} results already generated..."
-            else
-                python generate/generate_data_model_random_center.py --output ${FILENAME} --interval "0,${size}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
-                python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
-
-                python others/save_model_result_in_md_maxwell.py --interval "0,${size}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}
-            fi
-        done
-    done
-done
-

data_processing/generateAndTrain_maxwell_custom_filters_split.sh

@@ -1,48 +0,0 @@
-#! bin/bash
-
-if [ -z "$1" ]
-  then
-    echo "No argument supplied"
-    echo "Need of vector size"
-    exit 1
-fi
-
-if [ -z "$2" ]
-  then
-    echo "No argument supplied"
-    echo "Need of feature information"
-    exit 1
-fi
-
-result_filename="results/models_comparisons.csv"
-VECTOR_SIZE=200
-size=$1
-feature=$2
-
-# selection of four scenes (only maxwell)
-scenes="A, D, G, H"
-
-for nb_zones in {4,6,8,10,12}; do
-    for mode in {"svd","svdn","svdne"}; do
-        for model in {"svm_model","ensemble_model","ensemble_model_v2"}; do
-
-            FILENAME="data/${model}_N${size}_B0_E${size}_nb_zones_${nb_zones}_${feature}_${mode}"
-            MODEL_NAME="${model}_N${size}_B0_E${size}_nb_zones_${nb_zones}_${feature}_${mode}"
-            CUSTOM_MIN_MAX_FILENAME="N${size}_B0_E${size}_nb_zones_${nb_zones}_${feature}_${mode}_min_max"
-
-            echo $FILENAME
-
-            # only compute if necessary (perhaps server will fall.. Just in case)
-            if grep -q "${MODEL_NAME}" "${result_filename}"; then
-
-                echo "${MODEL_NAME} results already generated..."
-            else
-                python generate/generate_data_model_random_split.py --output ${FILENAME} --interval "0,${size}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
-                python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
-
-                python others/save_model_result_in_md_maxwell.py --interval "0,${size}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}
-            fi
-        done
-    done
-done
-

data_processing/generateAndTrain_maxwell_custom_split.sh

@@ -1,74 +0,0 @@
-#! bin/bash
-
-if [ -z "$1" ]
-  then
-    echo "No argument supplied"
-    echo "Need of vector size"
-    exit 1
-fi
-
-if [ -z "$2" ]
-  then
-    echo "No argument supplied"
-    echo "Need of feature information"
-    exit 1
-fi
-
-result_filename="results/models_comparisons.csv"
-VECTOR_SIZE=200
-size=$1
-feature=$2
-
-# selection of four scenes (only maxwell)
-scenes="A, D, G, H"
-
-half=$(($size/2))
-start=-$half
-for counter in {0..4}; do
-    end=$(($start+$size))
-
-    if [ "$end" -gt "$VECTOR_SIZE" ]; then
-        start=$(($VECTOR_SIZE-$size))
-        end=$(($VECTOR_SIZE))
-    fi
-
-    if [ "$start" -lt "0" ]; then
-        start=$((0))
-        end=$(($size))
-    fi
-
-    for nb_zones in {4,6,8,10,12}; do
-
-        echo $start $end
-
-        for mode in {"svd","svdn","svdne"}; do
-            for model in {"svm_model","ensemble_model","ensemble_model_v2"}; do
-
-                FILENAME="data/${model}_N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${feature}_${mode}"
-                MODEL_NAME="${model}_N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${feature}_${mode}"
-                CUSTOM_MIN_MAX_FILENAME="N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${feature}_${mode}_min_max"
-
-                echo $FILENAME
-
-                # only compute if necessary (perhaps server will fall.. Just in case)
-                if grep -q "${MODEL_NAME}" "${result_filename}"; then
-
-                    echo "${MODEL_NAME} results already generated..."
-                else
-                    python generate/generate_data_model_random_split.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
-                    python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
-
-                    #python prediction/predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
-                    python others/save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}
-                fi
-            done
-        done
-    done
-
-    if [ "$counter" -eq "0" ]; then
-        start=$(($start+50-$half))
-    else
-        start=$(($start+50))
-    fi
-
-done

run/runAll_maxwell.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -19,6 +34,6 @@ fi
 for size in {"4","8","16","26","32","40"}; do
 
     for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2"}; do
-        bash data_processing/generateAndTrain_maxwell.sh ${size} ${metric}
+        bash data_processing/generateAndTrain_maxwell.sh ${size} ${metric} ${data}
     done
 done

run/runAll_maxwell_area.sh

@@ -2,8 +2,22 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -13,7 +27,6 @@ if [ "${erased}" == "Y" ]; then
 
     # add of header
     echo 'model_name; vector_size; start_index; end; nb_zones; feature; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
 fi
 
 feature="sub_blocks_area"
@@ -40,7 +53,7 @@ for nb_zones in {4,6,8,10,12}; do
 
                 echo "${MODEL_NAME} results already generated..."
             else
-                python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
                 python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                 python others/save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}

run/runAll_maxwell_area_normed.sh

@@ -2,8 +2,22 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -40,7 +54,7 @@ for nb_zones in {4,6,8,10,12}; do
 
                 echo "${MODEL_NAME} results already generated..."
             else
-                python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
                 python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                 python others/save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}

run/runAll_maxwell_custom.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -19,6 +34,6 @@ fi
 for size in {"4","8","16","26","32","40"}; do
 
     for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
-        bash data_processing/generateAndTrain_maxwell_custom.sh ${size} ${metric}
+        bash data_processing/generateAndTrain_maxwell_custom.sh ${size} ${metric} ${data}
     done
 done

run/runAll_maxwell_custom_center.sh

@@ -1,24 +0,0 @@
-#! bin/bash
-
-# erase "results/models_comparisons.csv" file and write new header
-file_path='results/models_comparisons.csv'
-
-erased=$1
-
-if [ "${erased}" == "Y" ]; then
-    echo "Previous data file erased..."
-    rm ${file_path}
-    mkdir -p results
-    touch ${file_path}
-
-    # add of header
-    echo 'model_name; vector_size; start; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
-fi
-
-for size in {"4","8","16","26","32","40"}; do
-
-    for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
-        bash data_processing/generateAndTrain_maxwell_custom_center.sh ${size} ${metric}
-    done
-done

run/runAll_maxwell_custom_filters.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -20,6 +35,6 @@ for size in {"4","8","16","26","32","40","60","80"}; do
 
 #    for metric in {"highest_sv_std_filters","lowest_sv_std_filters","highest_wave_sv_std_filters","lowest_sv_std_filters","highest_sv_std_filters_full","lowest_sv_std_filters_full","highest_sv_entropy_std_filters","lowest_sv_entropy_std_filters"}; do
     for metric in {"highest_sv_entropy_std_filters","lowest_sv_entropy_std_filters"}; do
-        bash data_processing/generateAndTrain_maxwell_custom_filters.sh ${size} ${metric} &
+        bash data_processing/generateAndTrain_maxwell_custom_filters.sh ${size} ${metric} ${data} &
     done
 done

run/runAll_maxwell_custom_filters_center.sh

@@ -1,25 +0,0 @@
-#! bin/bash
-
-# erase "results/models_comparisons.csv" file and write new header
-file_path='results/models_comparisons.csv'
-
-erased=$1
-
-if [ "${erased}" == "Y" ]; then
-    echo "Previous data file erased..."
-    rm ${file_path}
-    mkdir -p results
-    touch ${file_path}
-
-    # add of header
-    echo 'model_name; vector_size; start; end; nb_zones; metric; mode; train_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
-fi
-
-for size in {"4","8","16","26","32","40","60","80"}; do
-
-#     for metric in {"highest_sv_std_filters","lowest_sv_std_filters","highest_wave_sv_std_filters","lowest_sv_std_filters","highest_sv_std_filters_full","lowest_sv_std_filters_full","highest_sv_entropy_std_filters","lowest_sv_entropy_std_filters"}; do
-     for metric in {"highest_sv_entropy_std_filters","lowest_sv_entropy_std_filters"}; do
-        bash data_processing/generateAndTrain_maxwell_custom_filters_center.sh ${size} ${metric} &
-    done
-done

run/runAll_maxwell_custom_filters_split.sh

@@ -1,25 +0,0 @@
-#! bin/bash
-
-# erase "results/models_comparisons.csv" file and write new header
-file_path='results/models_comparisons.csv'
-
-erased=$1
-
-if [ "${erased}" == "Y" ]; then
-    echo "Previous data file erased..."
-    rm ${file_path}
-    mkdir -p results
-    touch ${file_path}
-
-    # add of header
-    echo 'model_name; vector_size; start; end; nb_zones; metric; mode; train_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
-fi
-
-for size in {"4","8","16","26","32","40","60","80"}; do
-
-#    for metric in {"highest_sv_std_filters","lowest_sv_std_filters","highest_wave_sv_std_filters","lowest_sv_std_filters","highest_sv_std_filters_full","lowest_sv_std_filters_full","highest_sv_entropy_std_filters","lowest_sv_entropy_std_filters"}; do
-    for metric in {"highest_sv_entropy_std_filters","lowest_sv_entropy_std_filters"}; do
-        bash data_processing/generateAndTrain_maxwell_custom_filters_split.sh ${size} ${metric} &
-    done
-done

run/runAll_maxwell_custom_filters_stats.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -13,10 +28,9 @@ if [ "${erased}" == "Y" ]; then
 
     # add of header
     echo 'model_name; vector_size; start; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
 fi
 
 size=26
 metric="filters_statistics"
 
-bash data_processing/generateAndTrain_maxwell_custom_filters.sh ${size} ${metric} &
+bash data_processing/generateAndTrain_maxwell_custom_filters.sh ${size} ${metric} ${data} &

run/runAll_maxwell_custom_filters_stats_center.sh

@@ -1,22 +0,0 @@
-#! bin/bash
-
-# erase "results/models_comparisons.csv" file and write new header
-file_path='results/models_comparisons.csv'
-
-erased=$1
-
-if [ "${erased}" == "Y" ]; then
-    echo "Previous data file erased..."
-    rm ${file_path}
-    mkdir -p results
-    touch ${file_path}
-
-    # add of header
-    echo 'model_name; vector_size; start; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
-fi
-
-size=26
-metric="filters_statistics"
-
-bash data_processing/generateAndTrain_maxwell_custom_filters_center.sh ${size} ${metric} &

run/runAll_maxwell_custom_filters_stats_split.sh

@@ -1,22 +0,0 @@
-#! bin/bash
-
-# erase "results/models_comparisons.csv" file and write new header
-file_path='results/models_comparisons.csv'
-
-erased=$1
-
-if [ "${erased}" == "Y" ]; then
-    echo "Previous data file erased..."
-    rm ${file_path}
-    mkdir -p results
-    touch ${file_path}
-
-    # add of header
-    echo 'model_name; vector_size; start; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
-fi
-
-size=26
-metric="filters_statistics"
-
-bash data_processing/generateAndTrain_maxwell_custom_filters_split.sh ${size} ${metric} &

run/runAll_maxwell_custom_split.sh

@@ -1,24 +0,0 @@
-#! bin/bash
-
-# erase "results/models_comparisons.csv" file and write new header
-file_path='results/models_comparisons.csv'
-
-erased=$1
-
-if [ "${erased}" == "Y" ]; then
-    echo "Previous data file erased..."
-    rm ${file_path}
-    mkdir -p results
-    touch ${file_path}
-
-    # add of header
-    echo 'model_name; vector_size; start; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
-
-fi
-
-for size in {"4","8","16","26","32","40"}; do
-
-    for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
-        bash data_processing/generateAndTrain_maxwell_custom_split.sh ${size} ${metric}
-    done
-done

run/runAll_maxwell_keras.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -43,11 +58,10 @@ for feature in {"sub_blocks_stats","sub_blocks_stats_reduced","sub_blocks_area",
 
                 echo "${MODEL_NAME} results already generated..."
             else
-                echo "test"
-                #python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
-                #python deep_network_keras_svd.py --data ${FILENAME} --output ${MODEL_NAME} --size ${end_index}
+                python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                python deep_network_keras_svd.py --data ${FILENAME} --output ${MODEL_NAME} --size ${end_index}
 
-                #python others/save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.json" --mode "${mode}" --feature ${feature}
+                python others/save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.json" --mode "${mode}" --feature ${feature}
             fi
         done
     done

run/runAll_maxwell_mscn_var.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -43,7 +58,7 @@ for nb_zones in {4,6,8,10,12}; do
 
                     echo "${MODEL_NAME} results already generated..."
                 else
-                    python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                    python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
                     python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                     python others/save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}

run/runAll_maxwell_sub_blocks_stats.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -40,7 +55,7 @@ for nb_zones in {4,6,8,10,12}; do
 
                 echo "${MODEL_NAME} results already generated..."
             else
-                python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
                 python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                 python others/save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}

run/runAll_maxwell_sub_blocks_stats_reduced.sh

@@ -2,8 +2,23 @@
 
 # erase "results/models_comparisons.csv" file and write new header
 file_path='results/models_comparisons.csv'
+list="all, center, split"
 
-erased=$1
+if [ -z "$1" ]
+  then
+    echo "No argument supplied"
+    echo "Need argument from [${list}]"
+    exit 1
+fi
+
+if [[ "$1" =~ ^(all|center|split)$ ]]; then
+    echo "$1 is in the list"
+else
+    echo "$1 is not in the list"
+fi
+
+data=$1
+erased=$2
 
 if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
@@ -40,7 +55,7 @@ for nb_zones in {4,6,8,10,12}; do
 
                 echo "${MODEL_NAME} results already generated..."
             else
-                python generate/generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                python generate/generate_data_model_random_${data}.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --feature ${feature} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
                 python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
 
                 python others/save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --feature ${feature}