Merge branch 'release/v0.0.3'

README.md

@@ -4,7 +4,11 @@
 
 Noise detection on synthesis images with 26 attributes obtained using few filters. 
 
-TODO : list filters used
+Filters list:
+- wiener
+- median
+- gaussian
+- wavelet
 
 ## Requirements
 

generate_all_data.py

@@ -7,7 +7,7 @@ Created on Fri Sep 14 21:02:42 2018
 """
 
 from __future__ import print_function
-import sys, os, getopt
+import sys, os, argparse
 import numpy as np
 import random
 import time
@@ -180,33 +180,15 @@ def generate_data_svd(data_type, mode):
 
 def main():
 
-    # default value of p_step
-    p_step = 1
-
-    # TODO : use of argparse
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('python generate_all_data.py --metric all')
-        print('python generate_all_data.py --metric lab')
-        print('python generate_all_data.py --metric lab')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "hms", ["help=", "metric="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python generate_all_data.py --metric all')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python generate_all_data.py --metric all')
-            sys.exit()
-        elif o in ("-m", "--metric"):
-            p_metric = a
-
-            if p_metric != 'all' and p_metric not in metric_choices:
-                assert False, "Invalid metric choice"
-        else:
-            assert False, "unhandled option"
+    parser = argparse.ArgumentParser(description="Compute and prepare data of metric of all scenes (keep in memory min and max value found)")
+
+    parser.add_argument('--metric', type=str, 
+                                    help="metric choice in order to compute data (use 'all' if all metrics are needed)", 
+                                    choices=metric_choices)
+
+    args = parser.parse_args()
+
+    p_metric = args.metric
 
     # generate all or specific metric data
     if p_metric == 'all':

list_files.sh

@@ -1,12 +0,0 @@
-search_dir=$1
-sentence=$2
-
-for entry in "$search_dir"/*
-do
-    if [ -f $entry ]; then
-        if grep -q "$sentence" "$entry"; then
-            echo "$entry"
-        fi
-    fi
-done
-

predict_seuil_expe.py

@@ -5,7 +5,7 @@ import numpy as np
 from ipfml import processing, utils
 from PIL import Image
 
-import sys, os, getopt
+import sys, os, argparse
 import subprocess
 import time
 
@@ -20,6 +20,8 @@ threshold_map_folder      = cfg.threshold_map_folder
 threshold_map_file_prefix = cfg.threshold_map_folder + "_"
 
 zones                     = cfg.zones_indices
+normalization_choices     = cfg.normalization_choices
+metric_choices            = cfg.metric_choices_labels
 
 tmp_filename              = '/tmp/__model__img_to_predict.png'
 
@@ -29,43 +31,25 @@ def main():
 
     p_custom = False
 
-    # TODO : use of argparse
-    
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('python predict_seuil_expe.py --interval "0,20" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ht:m:o:l:c", ["help=", "interval=", "model=", "mode=", "metric=" "limit_detection=", "custom="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python predict_seuil_expe.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python predict_seuil_expe.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-            sys.exit()
-        elif o in ("-t", "--interval"):
-            p_interval = a
-        elif o in ("-mo", "--model"):
-            p_model_file = a
-        elif o in ("-o", "--mode"):
-            p_mode = a
-
-            if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
-                assert False, "Mode not recognized"
-
-        elif o in ("-me", "--metric"):
-            p_metric = a
-        elif o in ("-l", "--limit_detection"):
-            p_limit = int(a)
-        elif o in ("-c", "--custom"):
-            p_custom = a
-        else:
-            assert False, "unhandled option"
+    parser = argparse.ArgumentParser(description="Script which predicts threshold using specific model")
 
-    scenes = os.listdir(scenes_path)
+    parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
+    parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
+    parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    #parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
+
+    args = parser.parse_args()
 
+    p_interval   = list(map(int, args.interval.split(',')))
+    p_model_file = args.model
+    p_mode       = args.mode
+    p_metric     = args.metric
+    #p_limit      = args.limit
+    p_custom     = args.custom
+
+    scenes = os.listdir(scenes_path)
     scenes = [s for s in scenes if not min_max_filename in s]
 
     # go ahead each scenes

predict_seuil_expe_maxwell.py

@@ -5,7 +5,7 @@ import numpy as np
 from ipfml import processing
 from PIL import Image
 
-import sys, os, getopt
+import sys, os, argparse
 import subprocess
 import time
 
@@ -22,6 +22,8 @@ threshold_map_file_prefix = cfg.threshold_map_folder + "_"
 
 zones                     = cfg.zones_indices
 maxwell_scenes            = cfg.maxwell_scenes_names
+normalization_choices     = cfg.normalization_choices
+metric_choices            = cfg.metric_choices_labels
 
 tmp_filename              = '/tmp/__model__img_to_predict.png'
 
@@ -32,43 +34,25 @@ def main():
     # by default..
     p_custom = False
 
-    # TODO : use of argparse
-
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('python predict_seuil_expe_maxwell.py --interval "0,20" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ht:m:o:l:c", ["help=", "interval=", "model=", "mode=", "metric=", "limit_detection=", "custom="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python predict_seuil_expe_maxwell.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python predict_seuil_expe_maxwell.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-            sys.exit()
-        elif o in ("-t", "--interval"):
-            p_interval = a
-        elif o in ("-m", "--model"):
-            p_model_file = a
-        elif o in ("-o", "--mode"):
-            p_mode = a
-
-            if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
-                assert False, "Mode not recognized"
-
-        elif o in ("-m", "--metric"):
-            p_metric = a
-        elif o in ("-l", "--limit_detection"):
-            p_limit = int(a)
-        elif o in ("-c", "--custom"):
-            p_custom = a
-        else:
-            assert False, "unhandled option"
+    parser = argparse.ArgumentParser(description="Script which predicts threshold using specific model")
 
-    scenes = os.listdir(scenes_path)
+    parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
+    parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
+    parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    #parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
+
+    args = parser.parse_args()
 
+    p_interval   = list(map(int, args.interval.split(',')))
+    p_model_file = args.model
+    p_mode       = args.mode
+    p_metric     = args.metric
+    #p_limit      = args.limit
+    p_custom     = args.custom
+
+    scenes = os.listdir(scenes_path)
     scenes = [s for s in scenes if s in maxwell_scenes]
 
     # go ahead each scenes

predict_seuil_expe_maxwell_curve.py

@@ -5,7 +5,7 @@ import numpy as np
 from ipfml import processing
 from PIL import Image
 
-import sys, os, getopt
+import sys, os, argparse
 import subprocess
 import time
 
@@ -21,6 +21,8 @@ threshold_map_file_prefix = cfg.threshold_map_folder + "_"
 
 zones                     = cfg.zones_indices
 maxwell_scenes            = cfg.maxwell_scenes_names
+normalization_choices     = cfg.normalization_choices
+metric_choices            = cfg.metric_choices_labels
 
 simulation_curves_zones   = "simulation_curves_zones_"
 tmp_filename              = '/tmp/__model__img_to_predict.png'
@@ -31,44 +33,26 @@ current_dirpath = os.getcwd()
 def main():
 
     p_custom = False
+        
+    parser = argparse.ArgumentParser(description="Script which predicts threshold using specific model")
 
-    # TODO : use of argparse
-    
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('python predict_seuil_expe_maxwell_curve.py --interval "0,20" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ht:m:o:l:c", ["help=", "interval=", "model=", "mode=", "metric=", "limit_detection=", "custom="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python predict_seuil_expe_maxwell_curve.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python predict_seuil_expe_maxwell_curve.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx --custom min_max_filename')
-            sys.exit()
-        elif o in ("-t", "--interval"):
-            p_interval = a
-        elif o in ("-m", "--model"):
-            p_model_file = a
-        elif o in ("-o", "--mode"):
-            p_mode = a
-
-            if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
-                assert False, "Mode not recognized"
-
-        elif o in ("-m", "--metric"):
-            p_metric = a
-        elif o in ("-l", "--limit_detection"):
-            p_limit = int(a)
-        elif o in ("-c", "--custom"):
-            p_custom = a
-        else:
-            assert False, "unhandled option"
+    parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
+    parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
+    parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    #parser.add_argument('--limit_detection', type=int, help='Specify number of same prediction to stop threshold prediction', default=2)
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
 
-    scenes = os.listdir(scenes_path)
+    args = parser.parse_args()
+
+    p_interval   = list(map(int, args.interval.split(',')))
+    p_model_file = args.model
+    p_mode       = args.mode
+    p_metric     = args.metric
+    #p_limit      = args.limit
+    p_custom     = args.custom
 
+    scenes = os.listdir(scenes_path)
     scenes = [s for s in scenes if s in maxwell_scenes]
 
     print(scenes)

prediction_scene.py

@@ -11,7 +11,7 @@ from keras import backend as K
 from keras.models import model_from_json
 from keras.wrappers.scikit_learn import KerasClassifier
 
-import sys, os, getopt
+import sys, os, argparse
 import json
 
 from modules.utils import config as cfg
@@ -19,33 +19,20 @@ from modules.utils import config as cfg
 output_model_folder = cfg.saved_models_folder
 
 def main():
-
-    # TODO : use of argparse
     
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('python prediction_scene.py --data xxxx.csv --model xxxx.joblib --output xxxx --scene xxxx')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "hd:o:s", ["help=", "data=", "model=", "output=", "scene="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python prediction_scene.py --data xxxx.csv --model xxxx.joblib --output xxxx --scene xxxx')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python prediction_scene.py --data xxxx.csv --model xxxx.joblib --output xxxx --scene xxxx')
-            sys.exit()
-        elif o in ("-d", "--data"):
-            p_data_file = a
-        elif o in ("-m", "--model"):
-            p_model_file = a
-        elif o in ("-o", "--output"):
-            p_output = a
-        elif o in ("-s", "--scene"):
-            p_scene = a
-        else:
-            assert False, "unhandled option"
+    parser = argparse.ArgumentParser(description="Give model performance on specific scene")
+
+    parser.add_argument('--data', type=str, help='dataset filename prefix of specific scene (without .train and .test)')
+    parser.add_argument('--model', type=str, help='saved model (Keras or SKlearn) filename with extension')
+    parser.add_argument('--output', type=str, help="filename to store predicted and performance model obtained on scene")
+    parser.add_argument('--scene', type=str, help="scene indice to predict", choices=cfg.scenes_indices)
+
+    args = parser.parse_args()
+
+    p_data_file  = args.data
+    p_model_file = args.model
+    p_output     = args.output
+    p_scene      = args.scene
 
     if '.joblib' in p_model_file:
         kind_model = 'sklearn'

train_model.py

@@ -11,7 +11,7 @@ from sklearn.model_selection import cross_val_score
 
 import numpy as np
 import pandas as pd
-import sys, os, getopt
+import sys, os, argparse
 
 from modules.utils import config as cfg
 from modules import models as mdl
@@ -25,33 +25,17 @@ output_model_folder = os.path.join(current_dirpath, saved_models_folder)
 
 def main():
 
-    # TODO : use argparse
-    
-    if len(sys.argv) <= 2:
-        print('python train_model.py --data xxxx --output xxxx --choice svm_model')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "hd:o:c", ["help=", "data=", "output=", "choice="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python train_model.py --data xxxx --output xxxx --choice svm_model')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python train_model.py --data xxxx --output xxxx --choice svm_model')
-            sys.exit()
-        elif o in ("-d", "--data"):
-            p_data_file = a
-        elif o in ("-o", "--output"):
-            p_output = a
-        elif o in ("-c", "--choice"):
-            p_choice = a
-
-            if not p_choice in models_list:
-                assert False, "Unknown model choice"
-
-        else:
-            assert False, "unhandled option"
+    parser = argparse.ArgumentParser(description="Train SKLearn model and save it into .joblib file")
+
+    parser.add_argument('--data', type=str, help='dataset filename prefix (without .train and .test)')
+    parser.add_argument('--output', type=str, help='output file name desired for model (without .joblib extension)')
+    parser.add_argument('--choice', type=str, help='model choice from list of choices', choices=models_list)
+
+    args = parser.parse_args()
+
+    p_data_file = args.data
+    p_output    = args.output
+    p_choice    = args.choice
 
     if not os.path.exists(output_model_folder):
         os.makedirs(output_model_folder)
@@ -131,7 +115,6 @@ def main():
     val_f1 = f1_score(y_val, y_val_model)
     test_f1 = f1_score(y_test, y_test_model)
 
-
     ###################
     # 5. Output : Print and write all information in csv
     ###################
@@ -143,7 +126,6 @@ def main():
     print("Test: ", val_accuracy)
     print("Test F1: ", test_f1)
 
-
     ##################
     # 6. Save model : create path if not exists
     ##################