Browse Source

Update of predict svd script

Jérôme BUISINE 1 year ago
parent
commit
51606962b8

File diff suppressed because it is too large
+ 116 - 50
analysis/mscn_analysis.ipynb


+ 2 - 2
modules/utils/config.py

@@ -35,7 +35,7 @@ cycle_scenes_indices            = ['E', 'I']
 normalization_choices           = ['svd', 'svdn', 'svdne']
 zones_indices                   = np.arange(16)
 
-metric_choices_labels           = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed', 'mscn_var_4', 'mscn_var_16', 'mscn_var_64']
+metric_choices_labels           = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed', 'mscn_var_4', 'mscn_var_16', 'mscn_var_64', 'mscn_var_16_max', 'mscn_var_64_max']
 
-keras_epochs                    = 1000
+keras_epochs                    = 500
 keras_batch                     = 32

+ 16 - 0
modules/utils/data.py

@@ -185,6 +185,22 @@ def get_svd_data(data_type, block):
 
         data = _get_mscn_variance(block, (25, 25))
 
+    if data_type == 'mscn_var_16_max':
+
+        data = _get_mscn_variance(block, (50, 50))
+        data = np.asarray(data)
+        size = int(len(data) / 4)
+        indices = data.argsort()[-size:][::-1]
+        data = data[indices]
+
+    if data_type == 'mscn_var_64_max':
+
+        data = _get_mscn_variance(block, (25, 25))
+        data = np.asarray(data)
+        size = int(len(data) / 4)
+        indices = data.argsort()[-size:][::-1]
+        data = data[indices]
+
     return data
 
 def _get_mscn_variance(block, sub_block_size=(50, 50)):

+ 6 - 4
predict_noisy_image_svd.py

@@ -5,7 +5,9 @@ import numpy as np
 from ipfml import processing, utils
 from PIL import Image
 
-import sys, os, argparse
+import sys, os, argparse, json
+
+from keras.models import model_from_json
 
 from modules.utils import config as cfg
 from modules.utils import data as dt
@@ -47,7 +49,7 @@ def main():
     if 'corr' in p_model_file:
         corr_model = True
 
-        indices_corr_path = os.path.join(cfg.correlation_indices_folder, p_model_file.split('.')[0] + '.csv')
+        indices_corr_path = os.path.join(cfg.correlation_indices_folder, p_model_file.split('/')[1].replace('.json', '') + '.csv')
 
         with open(indices_corr_path, 'r') as f:
             data_corr_indices = [int(x) for x in f.readline().split(';') if x != '']
@@ -133,8 +135,8 @@ def main():
         prediction = model.predict([test_data])[0]
 
     if kind_model == 'keras':
-        test_data = test_data.reshape(len(test_data), 1)
-        prediction = model.predict_classes([test_data])[0]
+        test_data = np.asarray(test_data).reshape(1, len(test_data), 1)
+        prediction = model.predict_classes([test_data])[0][0]
 
     # output expected from others scripts
     print(prediction)

+ 2 - 2
runAll_maxwell_mscn_var.sh

@@ -23,12 +23,12 @@ end_index=4
 scenes="A, D, G, H"
 
 declare -A metrics_size
-metrics_size=( ["mscn_var_4"]=4 ["mscn_var_16"]=16 ["mscn_var_64"]=64)
+metrics_size=( ["mscn_var_4"]=4 ["mscn_var_16"]=16 ["mscn_var_64"]=64 ["mscn_var_16_max"]=4 ["mscn_var_64_max"]=16)
 
 for nb_zones in {4,6,8,10,12}; do
 
     for mode in {"svd","svdn","svdne"}; do
-        for metric in {"mscn_var_4","mscn_var_16","mscn_var_64"}; do
+        for metric in {"mscn_var_4","mscn_var_16","mscn_var_64","mscn_var_16_max","mscn_var_64_max"}; do
             for model in {"svm_model","ensemble_model","ensemble_model_v2"}; do
 
                 end_index=${metrics_size[${metric}]}

+ 7 - 3
run_maxwell_simulation_keras_corr_custom.sh

@@ -3,6 +3,9 @@
 # file which contains model names we want to use for simulation
 simulate_models="simulate_models_keras_corr.csv"
 
+start_index=0
+end_index=24
+
 # selection of four scenes (only maxwell)
 scenes="A, D, G, H"
 metric="lab"
@@ -19,13 +22,14 @@ for label in {"0","1"}; do
 
                     CUSTOM_MIN_MAX_FILENAME="N${size}_B${start_index}_E${end_index}_nb_zones_${nb_zones}_${metric}_${mode}_corr_L${label}_H${highest}_min_max"
 
+                    echo ${MODEL_NAME}
+
                     if grep -xq "${MODEL_NAME}" "${simulate_models}"; then
                         echo "Run simulation for model ${MODEL_NAME}"
 
-                        # by default regenerate model
-                        python generate_data_model_random.py --output ${FILENAME} --interval "${start_index},${end_index}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 40 --random 1 --custom ${CUSTOM_MIN_MAX_FILENAME}
+                        python generate_data_model_corr_random.py --output ${FILENAME} --n ${size} --highest ${highest} --label ${label} --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --renderer "maxwell" --step 10 --random 1
 
-                        python train_model.py --data ${FILENAME} --output ${MODEL_NAME} --choice ${model}
+                        python deep_network_keras_svd.py --data ${FILENAME} --output ${MODEL_NAME} --size ${size}
 
                         python predict_seuil_expe_maxwell_curve.py --interval "${start_index},${end_index}" --model "saved_models/${MODEL_NAME}.json" --mode "${mode}" --metric ${metric} --limit_detection '2' --custom ${CUSTOM_MIN_MAX_FILENAME}
 

+ 5 - 0
simulate_models_keras.csv

@@ -0,0 +1,5 @@
+deep_keras_N35_B0_E35_nb_zones_10_lab_svd_corr_L0_H1
+deep_keras_N25_B0_E25_nb_zones_10_lab_svdne_corr_L0_H1
+deep_keras_N30_B0_E30_nb_zones_6_lab_svdne_corr_L0_H1
+deep_keras_N5_B0_E5_nb_zones_12_lab_svdn_corr_L1_H1
+deep_keras_N25_B0_E25_nb_zones_6_lab_svdne_corr_L0_H1

+ 5 - 0
simulate_models_keras_corr.csv

@@ -0,0 +1,5 @@
+deep_keras_N35_B0_E35_nb_zones_10_lab_svd_corr_L0_H1
+deep_keras_N25_B0_E25_nb_zones_10_lab_svdne_corr_L0_H1
+deep_keras_N30_B0_E30_nb_zones_6_lab_svdne_corr_L0_H1
+deep_keras_N5_B0_E5_nb_zones_12_lab_svdn_corr_L1_H1
+deep_keras_N25_B0_E25_nb_zones_6_lab_svdne_corr_L0_H1