Add of mscn var runs

modules/utils/config.py

@@ -35,7 +35,7 @@ cycle_scenes_indices            = ['E', 'I']
 normalization_choices           = ['svd', 'svdn', 'svdne']
 zones_indices                   = np.arange(16)
 
-metric_choices_labels           = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed']
+metric_choices_labels           = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed', 'mscn_var_4', 'mscn_var_16', 'mscn_var_64']
 
 keras_epochs                    = 1000
 keras_batch                     = 32

modules/utils/data.py

@@ -173,8 +173,34 @@ def get_svd_data(data_type, block):
         # convert into numpy array after computing all stats
         data = np.asarray(data)
 
+    if data_type == 'mscn_var_4':
+
+        data = _get_mscn_variance(block, (100, 100))
+
+    if data_type == 'mscn_var_16':
+
+        data = _get_mscn_variance(block, (50, 50))
+
+    if data_type == 'mscn_var_64':
+
+        data = _get_mscn_variance(block, (25, 25))
+
     return data
 
+def _get_mscn_variance(block, sub_block_size=(50, 50)):
+
+    blocks = processing.divide_in_blocks(block, sub_block_size)
+
+    data = []
+
+    for block in blocks:
+        mscn_coefficients = processing.get_mscn_coefficients(block)
+        flat_coeff = mscn_coefficients.flatten()
+        data.append(np.var(flat_coeff))
+
+    return np.sort(data)
+
+
 def get_renderer_scenes_indices(renderer_name):
 
     if renderer_name not in renderer_choices:

runAll_maxwell_keras.sh

@@ -23,7 +23,8 @@ end_index=24
 # selection of four scenes (only maxwell)
 scenes="A, D, G, H"
 
-metrics_size=( ["sub_blocks_stats"]=24 ["sub_blocks_stats_reduced"]=20 ["sub_blocks_area"]=16 ["sub_blocks_area_normed"]=20)
+declare -A metrics_size
+metrics_size=( ["sub_blocks_stats"]="24" ["sub_blocks_stats_reduced"]="20" ["sub_blocks_area"]="16" ["sub_blocks_area_normed"]="20")
 
 for metric in {"sub_blocks_stats","sub_blocks_stats_reduced","sub_blocks_area","sub_blocks_area_normed"}; do
     for nb_zones in {4,6,8,10,12}; do
@@ -31,6 +32,7 @@ for metric in {"sub_blocks_stats","sub_blocks_stats_reduced","sub_blocks_area","
         for mode in {"svd","svdn","svdne"}; do
 
             end_index=${metrics_size[${metric}]}
+
             FILENAME="data/deep_keras_N${end_index}_B${start_index}_E${end_index}_nb_zones_${nb_zones}_${metric}_${mode}"
             MODEL_NAME="deep_keras_N${end_index}_B${start_index}_E${end_index}_nb_zones_${nb_zones}_${metric}_${mode}"
 
@@ -41,10 +43,11 @@ for metric in {"sub_blocks_stats","sub_blocks_stats_reduced","sub_blocks_area","
 
                 echo "${MODEL_NAME} results already generated..."
             else
-                python generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                echo "test"
-                python deep_network_keras_svd.py --data ${FILENAME} --output ${MODEL_NAME} --size ${end_index}
+                #python generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                #python deep_network_keras_svd.py --data ${FILENAME} --output ${MODEL_NAME} --size ${end_index}
 
-                python save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.json" --mode "${mode}" --metric ${metric}
+                #python save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.json" --mode "${mode}" --metric ${metric}
             fi
         done
     done

runAll_maxwell_mscn_var.sh

@@ -0,0 +1,56 @@
+#! bin/bash
+
+# erase "models_info/models_comparisons.csv" file and write new header
+file_path='models_info/models_comparisons.csv'
+
+erased=$1
+
+if [ "${erased}" == "Y" ]; then
+    echo "Previous data file erased..."
+    rm ${file_path}
+    mkdir -p models_info
+    touch ${file_path}
+
+    # add of header
+    echo 'model_name; vector_size; start_index; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; recall_train; roc_auc_train; F1_val; recall_val; roc_auc_val; F1_test; recall_test; roc_auc_test; F1_all; recall_all; roc_auc_all;' >> ${file_path}
+
+fi
+
+start_index=0
+end_index=4
+
+# selection of four scenes (only maxwell)
+scenes="A, D, G, H"
+
+declare -A metrics_size
+metrics_size=( ["mscn_var_4"]=4 ["mscn_var_16"]=16 ["mscn_var_64"]=64)
+
+for nb_zones in {4,6,8,10,12}; do
+
+    for mode in {"svd","svdn","svdne"}; do
+        for metric in {"mscn_var_4","mscn_var_16","mscn_var_64"}; do
+            for model in {"svm_model","ensemble_model","ensemble_model_v2"}; do
+
+                end_index=${metrics_size[${metric}]}
+
+                FILENAME="data/${model}_N${end_index}_B${start_index}_E${end_index}_nb_zones_${nb_zones}_${metric}_${mode}"
+                MODEL_NAME="${model}_N${end_index}_B${start_index}_E${end_index}_nb_zones_${nb_zones}_${metric}_${mode}"
+
+                echo $FILENAME
+
+                # only compute if necessary (perhaps server will fall.. Just in case)
+                if grep -q "${MODEL_NAME}" "${file_path}"; then
+
+                    echo "${MODEL_NAME} results already generated..."
+                else
+                    python generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
+                    python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
+
+                    python save_model_result_in_md_maxwell.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+                fi
+            done
+        done
+    done
+done
+
+