Merge branch 'release/v0.1.6'

.gitignore

@@ -4,6 +4,8 @@ saved_models/*
 threshold_map/*
 models_info/*
 
+simulate_models.csv
+
 fichiersSVD_light/*/*/*.csv
 fichiersSVD_light/*_min_max_values
 

README.md

@@ -14,12 +14,16 @@ python generate_all_data.py --metric all
 For noise detection, many metrics are available :
 - lab
 - mscn
+- mscn_revisited
 - low_bits_2
 - low_bits_4
+- low_bits_5
+- low_bits_6
+- low_bits_4_shifted_2
 
-You can also specify metric you want to compute :
+You can also specify metric you want to compute and image step to avoid some images :
 ```bash
-python generate_all_data.py --metric mscn
+python generate_all_data.py --metric mscn --step 50
 ```
 
 ## How to use
@@ -27,9 +31,9 @@ python generate_all_data.py --metric mscn
 ### Multiple folders and scripts are availables :
 
 
-- **fichiersSVD/\*** : all scene files information (zones of each scene, SVD descriptor files information and so on...).
-- **fichiersSVD_light/\*** : all scene files information (zones of each scene, SVD descriptor files information and so on...) but here with reduction of information for few scenes. Information used in our case.
+- **fichiersSVD_light/\*** : all scene files information (zones of each scene, SVD descriptor files information and so on...).
 - **models/*.py** : all models developed to predict noise in image.
+- **utils/** : contains all usefull script or modules.
 - **data/\*** : folder which will contain all *.train* & *.test* files in order to train model.
 - **saved_models/*.joblib** : all scikit learn models saved.
 - **models_info/*.md** : all markdown files generated to get quick information about model performance and prediction.
@@ -99,6 +103,18 @@ python prediction_scene.py --data path/to/xxxx.csv --model saved_model/xxxx.jobl
 ```
 **Remark** : *scene* parameter expected need to be the correct name of the Scene.
 
+### Visualize data
+
+All scripts with names **display_\*.py** are used to display data information or results.
+
+Just use --help option to get more information.
+
+### Simulate model on scene
+
+All scripts named **predict_seuil_expe\*.py** are used to simulate model prediction during rendering process.
+
+Once you have simulation done. Checkout your **threshold_map/%MODEL_NAME/simulation_curves_zones_\*** folder and use it with help of **display_simulation_curves.py** script.
+
 ## Others scripts
 
 ### Test model on all scene data
@@ -150,8 +166,7 @@ Parameters list :
 - **interval** : the interval of data you want to use from SVD vector.
 - **mode** : kind of data ['svd', 'svdn', 'svdne']; not normalize, normalize vector only and normalize together.
 
-
-Markdown file is saved using model name into **models_info** folder.
+Markdown file with all information is saved using model name into **models_info** folder.
 
 ### Others...
 
@@ -159,4 +174,4 @@ All others bash scripts are used to combine and run multiple model combinations.
 
 ## How to contribute
 
-This git project uses [git-flow](https://danielkummer.github.io/git-flow-cheatsheet/) implementation. You are free to contribute to it.git 
+This git project uses [git-flow](https://danielkummer.github.io/git-flow-cheatsheet/) implementation. You are free to contribute to it.git 

cnn_keras_svd.py

@@ -0,0 +1,210 @@
+from keras.preprocessing.image import ImageDataGenerator
+from keras.models import Sequential
+from keras.layers import Conv1D, MaxPooling1D
+from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
+from keras import backend as K
+import matplotlib.pyplot as plt
+
+from sklearn.utils import shuffle
+
+import numpy as np
+import pandas as pd
+
+from ipfml import image_processing
+from PIL import Image
+
+import sys, os, getopt
+import subprocess
+import time
+
+vector_size = 100
+epochs = 100
+batch_size = 24
+
+input_shape = (vector_size, 1)
+filename = "svd_model"
+
+def f1(y_true, y_pred):
+    def recall(y_true, y_pred):
+        """Recall metric.
+
+        Only computes a batch-wise average of recall.
+
+        Computes the recall, a metric for multi-label classification of
+        how many relevant items are selected.
+        """
+        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
+        recall = true_positives / (possible_positives + K.epsilon())
+        return recall
+
+    def precision(y_true, y_pred):
+        """Precision metric.
+
+        Only computes a batch-wise average of precision.
+
+        Computes the precision, a metric for multi-label classification of
+        how many selected items are relevant.
+        """
+        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
+        precision = true_positives / (predicted_positives + K.epsilon())
+        return precision
+    precision = precision(y_true, y_pred)
+    recall = recall(y_true, y_pred)
+    return 2*((precision*recall)/(precision+recall+K.epsilon()))
+
+def generate_model():
+
+    model = Sequential()
+
+    #model.add(Conv1D(128, (10), input_shape=input_shape))
+    #model.add(Activation('relu'))
+
+    #model.add(Conv1D(128, (10)))
+    #model.add(Activation('relu'))
+
+    #model.add(Conv1D(128, (10)))
+    #model.add(Activation('relu'))
+    #model.add(MaxPooling1D(pool_size=(2)))
+
+    #model.add(Conv1D(64, (10)))
+    #model.add(Activation('relu'))
+
+    #model.add(Conv1D(64, (10)))
+    #model.add(Activation('relu'))
+
+    #model.add(Conv1D(64, (10)))
+    #model.add(Activation('relu'))
+    #model.add(MaxPooling1D(pool_size=(2)))
+
+    #model.add(Conv1D(32, (10)))
+    #model.add(Activation('relu'))
+
+    #model.add(Conv1D(32, (10)))
+    #model.add(Activation('relu'))
+
+    #model.add(Conv1D(32, (10)))
+    #model.add(Activation('relu'))
+    #model.add(MaxPooling1D(pool_size=(2)))
+
+    model.add(Flatten(input_shape=input_shape))
+
+    #model.add(Dense(2048))
+    #model.add(Activation('relu'))
+    #model.add(BatchNormalization())
+    #model.add(Dropout(0.3))
+
+    model.add(Dense(1024))
+    model.add(Activation('relu'))
+    model.add(BatchNormalization())
+    model.add(Dropout(0.4))
+
+    model.add(Dense(512))
+    model.add(Activation('relu'))
+    model.add(BatchNormalization())
+    model.add(Dropout(0.4))
+
+    model.add(Dense(256))
+    model.add(Activation('relu'))
+    model.add(BatchNormalization())
+    model.add(Dropout(0.4))
+
+    model.add(Dense(128))
+    model.add(Activation('relu'))
+    model.add(BatchNormalization())
+    model.add(Dropout(0.4))
+
+    model.add(Dense(20))
+    model.add(Activation('relu'))
+    model.add(BatchNormalization())
+    model.add(Dropout(0.4))
+
+    model.add(Dense(1))
+    model.add(Activation('sigmoid'))
+
+    model.compile(loss='binary_crossentropy',
+                  optimizer='adam',
+                  metrics=['accuracy', f1])
+
+    return model
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python save_model_result_in_md.py --data filename')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hd", ["help=", "data="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python save_model_result_in_md.py --data filename')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python save_model_result_in_md.py --data filename')
+            sys.exit()
+        elif o in ("-d", "--data"):
+            p_datafile = a
+        else:
+            assert False, "unhandled option"
+
+    ###########################
+    # 1. Get and prepare data
+    ###########################
+    dataset_train = pd.read_csv(p_datafile + '.train', header=None, sep=";")
+    dataset_test = pd.read_csv(p_datafile + '.test', header=None, sep=";")
+
+    # default first shuffle of data
+    dataset_train = shuffle(dataset_train)
+    dataset_test = shuffle(dataset_test)
+
+    # get dataset with equal number of classes occurences
+    noisy_df_train = dataset_train[dataset_train.ix[:, 0] == 1]
+    not_noisy_df_train = dataset_train[dataset_train.ix[:, 0] == 0]
+    nb_noisy_train = len(noisy_df_train.index)
+
+    noisy_df_test = dataset_test[dataset_test.ix[:, 0] == 1]
+    not_noisy_df_test = dataset_test[dataset_test.ix[:, 0] == 0]
+    nb_noisy_test = len(noisy_df_test.index)
+
+    final_df_train = pd.concat([not_noisy_df_train[0:nb_noisy_train], noisy_df_train])
+    final_df_test = pd.concat([not_noisy_df_test[0:nb_noisy_test], noisy_df_test])
+
+    # shuffle data another time
+    final_df_train = shuffle(final_df_train)
+    final_df_test = shuffle(final_df_test)
+
+    final_df_train_size = len(final_df_train.index)
+    final_df_test_size = len(final_df_test.index)
+
+    # use of the whole data set for training
+    x_dataset_train = final_df_train.ix[:,1:]
+    x_dataset_test = final_df_test.ix[:,1:]
+
+    y_dataset_train = final_df_train.ix[:,0]
+    y_dataset_test = final_df_test.ix[:,0]
+
+    #######################
+    # 2. Getting model
+    #######################
+
+    model = generate_model()
+    model.summary()
+
+    #######################
+    # 3. Fit model : use of cross validation to fit model
+    #######################
+
+    # reshape input data
+    x_dataset_train = np.array(x_dataset_train).reshape(len(x_dataset_train), vector_size, 1)
+    x_dataset_test = np.array(x_dataset_test).reshape(len(x_dataset_test), vector_size, 1)
+
+    model.fit(x_dataset_train, y_dataset_train, epochs=epochs, batch_size=batch_size, validation_split=0.20)
+
+    score = model.evaluate(x_dataset_test, y_dataset_test, batch_size=batch_size)
+    print(score)
+
+if __name__== "__main__":
+    main()

display_bits_shifted.py

@@ -0,0 +1,60 @@
+from ipfml import image_processing
+from PIL import Image
+import numpy as np
+from ipfml import metrics
+from skimage import color
+
+import cv2
+
+nb_bits = 3
+max_nb_bits = 8
+
+low_bits_svd_values = []
+
+def open_and_display(path, i):
+
+    img = Image.open(path)
+
+    block_used = np.array(img)
+
+    low_bits_block = image_processing.rgb_to_LAB_L_bits(block_used, (i + 1, i + nb_bits + 1))
+
+    low_bits_svd = metrics.get_SVD_s(low_bits_block)
+
+    low_bits_svd = [b / low_bits_svd[0] for b in low_bits_svd]
+
+    low_bits_svd_values[i].append(low_bits_svd)
+
+path_noisy = '/home/jbuisine/Documents/Thesis/Development/NoiseDetection_In_SynthesisImages/fichiersSVD_light/Cuisine01/cuisine01_00050.png'
+path_threshold = '/home/jbuisine/Documents/Thesis/Development/NoiseDetection_In_SynthesisImages/fichiersSVD_light/Cuisine01/cuisine01_00400.png'
+path_ref = '/home/jbuisine/Documents/Thesis/Development/NoiseDetection_In_SynthesisImages/fichiersSVD_light/Cuisine01/cuisine01_01200.png'
+
+path_list = [path_noisy, path_threshold, path_ref]
+
+
+for i in range(0, max_nb_bits - nb_bits + 1):
+
+    low_bits_svd_values.append([])
+    for p in path_list:
+        open_and_display(p, i)
+
+import matplotlib.pyplot as plt
+
+# SVD
+# make a little extra space between the subplots
+
+fig=plt.figure(figsize=(8, 8))
+
+for id, l in enumerate(low_bits_svd_values):
+
+     fig.add_subplot(3, 3, (id + 1))
+     plt.plot(l[0], label='Noisy')
+     plt.plot(l[1], label='Threshold')
+     plt.plot(l[2], label='Reference')
+     plt.title('Low ' + str(nb_bits) + ' bits SVD shifted by ' + str(id))
+     plt.ylabel('Low ' + str(nb_bits) + ' bits SVD values')
+     plt.xlabel('Vector features')
+     plt.legend(bbox_to_anchor=(0.7, 1), loc=2, borderaxespad=0.2)
+     plt.ylim(0, 0.1)
+
+plt.show()

display_bits_shifted_scene.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep 14 21:02:42 2018
+
+@author: jbuisine
+"""
+
+from __future__ import print_function
+import sys, os, getopt
+import numpy as np
+import random
+import time
+import json
+
+from PIL import Image
+from ipfml import image_processing
+from ipfml import metrics
+from skimage import color
+import matplotlib.pyplot as plt
+
+config_filename   = "config"
+zone_folder       = "zone"
+min_max_filename  = "_min_max_values"
+
+# define all scenes values
+scenes_list = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
+scenes_indexes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
+choices = ['svd', 'svdn', 'svdne']
+path = '../fichiersSVD_light'
+zones = np.arange(16)
+seuil_expe_filename = 'seuilExpe'
+
+metric_choices = ['lab', 'mscn', 'mscn_revisited', 'low_bits_2', 'low_bits_3', 'low_bits_4']
+
+max_nb_bits = 8
+
+def display_data_scenes(nb_bits, p_scene):
+    """
+    @brief Method which generates all .csv files from scenes photos
+    @param path - path of scenes folder information
+    @return nothing
+    """
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(scenes):
+
+        if p_scene == folder_scene:
+            print(folder_scene)
+            scene_path = os.path.join(path, folder_scene)
+
+            config_file_path = os.path.join(scene_path, config_filename)
+
+            with open(config_file_path, "r") as config_file:
+                last_image_name = config_file.readline().strip()
+                prefix_image_name = config_file.readline().strip()
+                start_index_image = config_file.readline().strip()
+                end_index_image = config_file.readline().strip()
+                step_counter = int(config_file.readline().strip())
+
+            # construct each zones folder name
+            zones_folder = []
+
+            # get zones list info
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+
+                current_zone = "zone"+index_str
+                zones_folder.append(current_zone)
+
+            zones_images_data = []
+            threshold_info = []
+
+            for id_zone, zone_folder in enumerate(zones_folder):
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                current_counter_index = int(start_index_image)
+                end_counter_index = int(end_index_image)
+
+                # get threshold information
+                path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+                # open treshold path and get this information
+                with open(path_seuil, "r") as seuil_file:
+                    seuil_learned = int(seuil_file.readline().strip())
+                    threshold_info.append(seuil_learned)
+
+            # compute mean threshold values
+            mean_threshold = sum(threshold_info) / float(len(threshold_info))
+
+            print(mean_threshold, "mean threshold found")
+            threshold_image_found = False
+
+            # find appropriate mean threshold picture
+            while(current_counter_index <= end_counter_index and not threshold_image_found):
+
+                if mean_threshold < int(current_counter_index):
+                    current_counter_index_str = str(current_counter_index)
+
+                    while len(start_index_image) > len(current_counter_index_str):
+                        current_counter_index_str = "0" + current_counter_index_str
+
+                    threshold_image_found = True
+                    threshold_image_zone = current_counter_index_str
+
+                current_counter_index += step_counter
+
+            # all indexes of picture to plot
+            images_indexes = [start_index_image, threshold_image_zone, end_index_image]
+            images_data = []
+
+            print(images_indexes)
+
+            low_bits_svd_values = []
+
+            for i in range(0, max_nb_bits - nb_bits + 1):
+
+                low_bits_svd_values.append([])
+
+                for index in images_indexes:
+
+                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
+
+                    current_img = Image.open(img_path)
+
+                    block_used = np.array(current_img)
+
+                    low_bits_block = image_processing.rgb_to_LAB_L_bits(block_used, (i + 1, i + nb_bits + 1))
+                    low_bits_svd = metrics.get_SVD_s(low_bits_block)
+                    low_bits_svd = [b / low_bits_svd[0] for b in low_bits_svd]
+                    low_bits_svd_values[i].append(low_bits_svd)
+
+
+            fig=plt.figure(figsize=(8, 8))
+            fig.suptitle("Lab SVD " + str(nb_bits) +  " bits values shifted for " + p_scene + " scene", fontsize=20)
+
+            for id, data in enumerate(low_bits_svd_values):
+                fig.add_subplot(3, 3, (id + 1))
+                plt.plot(data[0], label='Noisy_' + start_index_image)
+                plt.plot(data[1], label='Threshold_' + threshold_image_zone)
+                plt.plot(data[2], label='Reference_' + end_index_image)
+                plt.ylabel('Lab SVD ' + str(nb_bits) + ' bits values shifted by ' + str(id), fontsize=14)
+                plt.xlabel('Vector features', fontsize=16)
+                plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)
+                plt.ylim(0, 0.1)
+            plt.show()
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python generate_all_data.py --bits 3 --scene A')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hb:s", ["help=", "bits=", "scene="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python generate_all_data.py --bits 4 --scene A')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python generate_all_data.py --bits 4 --scene A')
+            sys.exit()
+        elif o in ("-b", "--bits"):
+            p_bits = int(a)
+
+        elif o in ("-s", "--scene"):
+            p_scene = a
+
+            if p_scene not in scenes_indexes:
+                assert False, "Invalid metric choice"
+            else:
+                    p_scene = scenes_list[scenes_indexes.index(p_scene)]
+        else:
+            assert False, "unhandled option"
+
+
+    display_data_scenes(p_bits, p_scene)
+
+if __name__== "__main__":
+    main()

display_scenes_zones.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep 14 21:02:42 2018
+
+@author: jbuisine
+"""
+
+from __future__ import print_function
+import sys, os, getopt
+import numpy as np
+import random
+import time
+import json
+
+from PIL import Image
+from ipfml import image_processing
+from ipfml import metrics
+from skimage import color
+import matplotlib.pyplot as plt
+
+config_filename   = "config"
+zone_folder       = "zone"
+min_max_filename  = "_min_max_values"
+
+# define all scenes values
+scenes_list = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
+scenes_indexes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
+choices = ['svd', 'svdn', 'svdne']
+path = '../fichiersSVD_light'
+zones = np.arange(16)
+seuil_expe_filename = 'seuilExpe'
+
+metric_choices = ['lab', 'mscn', 'mscn_revisited', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6', 'low_bits_4_shifted_2']
+
+def display_data_scenes(data_type, p_scene, p_kind):
+    """
+    @brief Method which generates all .csv files from scenes photos
+    @param path - path of scenes folder information
+    @return nothing
+    """
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(scenes):
+
+        if p_scene == folder_scene:
+            print(folder_scene)
+            scene_path = os.path.join(path, folder_scene)
+
+            config_file_path = os.path.join(scene_path, config_filename)
+
+            with open(config_file_path, "r") as config_file:
+                last_image_name = config_file.readline().strip()
+                prefix_image_name = config_file.readline().strip()
+                start_index_image = config_file.readline().strip()
+                end_index_image = config_file.readline().strip()
+                step_counter = int(config_file.readline().strip())
+
+            # construct each zones folder name
+            zones_folder = []
+
+            # get zones list info
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+
+                current_zone = "zone"+index_str
+                zones_folder.append(current_zone)
+
+            zones_images_data = []
+            threshold_info = []
+
+            for id_zone, zone_folder in enumerate(zones_folder):
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                current_counter_index = int(start_index_image)
+                end_counter_index = int(end_index_image)
+
+                # get threshold information
+                path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+                # open treshold path and get this information
+                with open(path_seuil, "r") as seuil_file:
+                    seuil_learned = int(seuil_file.readline().strip())
+
+                threshold_image_found = False
+                while(current_counter_index <= end_counter_index and not threshold_image_found):
+
+                    if seuil_learned < int(current_counter_index):
+                        current_counter_index_str = str(current_counter_index)
+
+                        while len(start_index_image) > len(current_counter_index_str):
+                            current_counter_index_str = "0" + current_counter_index_str
+
+                        threshold_image_found = True
+                        threshold_image_zone = current_counter_index_str
+                        threshold_info.append(threshold_image_zone)
+
+                    current_counter_index += step_counter
+
+                # all indexes of picture to plot
+                images_indexes = [start_index_image, threshold_image_zone, end_index_image]
+                images_data = []
+
+                print(images_indexes)
+
+                for index in images_indexes:
+
+                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
+
+                    current_img = Image.open(img_path)
+                    img_blocks = image_processing.divide_in_blocks(current_img, (200, 200))
+
+                    # getting expected block id
+                    block = img_blocks[id_zone]
+
+                    # get data from mode
+                    # Here you can add the way you compute data
+                    if data_type == 'lab':
+
+                        block_file_path = '/tmp/lab_img.png'
+                        block.save(block_file_path)
+                        data = image_processing.get_LAB_L_SVD_s(Image.open(block_file_path))
+
+                    if data_type == 'mscn_revisited':
+
+                        img_mscn_revisited = image_processing.rgb_to_mscn(block)
+
+                        # save tmp as img
+                        img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
+                        mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
+                        img_output.save(mscn_revisited_file_path)
+                        img_block = Image.open(mscn_revisited_file_path)
+
+                        # extract from temp image
+                        data = metrics.get_SVD_s(img_block)
+
+                    if data_type == 'mscn':
+
+                        img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
+                        img_mscn = image_processing.calculate_mscn_coefficients(img_gray, 7)
+                        img_mscn_norm = image_processing.normalize_2D_arr(img_mscn)
+
+                        img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
+
+                        data = metrics.get_SVD_s(img_mscn_gray)
+
+                    if data_type == 'low_bits_6':
+
+                        low_bits_6 = image_processing.rgb_to_LAB_L_low_bits(block, 63)
+
+                        # extract from temp image
+                        data = metrics.get_SVD_s(low_bits_6)
+
+
+                    if data_type == 'low_bits_5':
+
+                        low_bits_5 = image_processing.rgb_to_LAB_L_low_bits(block, 31)
+
+                        # extract from temp image
+                        data = metrics.get_SVD_s(low_bits_5)
+
+
+                    if data_type == 'low_bits_4':
+
+                        low_bits_4 = image_processing.rgb_to_LAB_L_low_bits(block)
+
+                        # extract from temp image
+                        data = metrics.get_SVD_s(low_bits_4)
+
+                    if data_type == 'low_bits_3':
+
+                        low_bits_3 = image_processing.rgb_to_LAB_L_low_bits(block, 7)
+
+                        # extract from temp image
+                        data = metrics.get_SVD_s(low_bits_3)
+
+                    if data_type == 'low_bits_2':
+
+                        low_bits_2 = image_processing.rgb_to_LAB_L_low_bits(block, 3)
+
+                        # extract from temp image
+                        data = metrics.get_SVD_s(low_bits_2)
+
+                    ##################
+                    # Data mode part #
+                    ##################
+
+                    # modify data depending mode
+
+                    if p_kind == 'svdn':
+                        data = image_processing.normalize_arr(data)
+
+                    if p_kind == 'svdne':
+                        path_min_max = os.path.join(path, data_type + min_max_filename)
+
+                        with open(path_min_max, 'r') as f:
+                            min_val = float(f.readline())
+                            max_val = float(f.readline())
+
+                        data = image_processing.normalize_arr_with_range(data, min_val, max_val)
+
+                    # append of data
+                    images_data.append(data)
+
+                zones_images_data.append(images_data)
+
+            fig=plt.figure(figsize=(8, 8))
+            fig.suptitle(data_type + " values for " + p_scene + " scene (normalization : " + p_kind + ")", fontsize=20)
+
+            for id, data in enumerate(zones_images_data):
+                fig.add_subplot(4, 4, (id + 1))
+                plt.plot(data[0], label='Noisy_' + start_index_image)
+                plt.plot(data[1], label='Threshold_' + threshold_info[id])
+                plt.plot(data[2], label='Reference_' + end_index_image)
+                plt.ylabel(data_type + ' SVD, ZONE_' + str(id + 1), fontsize=18)
+                plt.xlabel('Vector features', fontsize=18)
+                plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=18)
+                plt.ylim(0, 0.1)
+            plt.show()
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python generate_all_data.py --metric all --scene A --kind svdn')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hm:s:k", ["help=", "metric=", "scene=", "kind="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python generate_all_data.py --metric all --scene A --kind svdn')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python generate_all_data.py --metric all --scene A --kind svdn')
+            sys.exit()
+        elif o in ("-m", "--metric"):
+            p_metric = a
+
+            if p_metric != 'all' and p_metric not in metric_choices:
+                assert False, "Invalid metric choice"
+        elif o in ("-s", "--scene"):
+            p_scene = a
+
+            if p_scene not in scenes_indexes:
+                assert False, "Invalid metric choice"
+            else:
+                p_scene = scenes_list[scenes_indexes.index(p_scene)]
+        elif o in ("-k", "--kind"):
+            p_kind = a
+
+            if p_kind not in choices:
+                assert False, "Invalid metric choice"
+        else:
+            assert False, "unhandled option"
+
+
+    display_data_scenes(p_metric, p_scene, p_kind)
+
+if __name__== "__main__":
+    main()

display_scenes_zones_shifted.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep 14 21:02:42 2018
+
+@author: jbuisine
+"""
+
+from __future__ import print_function
+import sys, os, getopt
+import numpy as np
+import random
+import time
+import json
+
+from PIL import Image
+from ipfml import image_processing
+from ipfml import metrics
+from skimage import color
+import matplotlib.pyplot as plt
+
+config_filename   = "config"
+zone_folder       = "zone"
+min_max_filename  = "_min_max_values"
+
+# define all scenes values
+scenes_list = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
+scenes_indexes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
+choices = ['svd', 'svdn', 'svdne']
+path = '../fichiersSVD_light'
+zones = np.arange(16)
+seuil_expe_filename = 'seuilExpe'
+
+max_nb_bits = 8
+
+def display_data_scenes(p_scene, p_bits, p_shifted):
+    """
+    @brief Method which generates all .csv files from scenes photos
+    @param path - path of scenes folder information
+    @return nothing
+    """
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(scenes):
+
+        if p_scene == folder_scene:
+            print(folder_scene)
+            scene_path = os.path.join(path, folder_scene)
+
+            config_file_path = os.path.join(scene_path, config_filename)
+
+            with open(config_file_path, "r") as config_file:
+                last_image_name = config_file.readline().strip()
+                prefix_image_name = config_file.readline().strip()
+                start_index_image = config_file.readline().strip()
+                end_index_image = config_file.readline().strip()
+                step_counter = int(config_file.readline().strip())
+
+            # construct each zones folder name
+            zones_folder = []
+
+            # get zones list info
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+
+                current_zone = "zone"+index_str
+                zones_folder.append(current_zone)
+
+            zones_images_data = []
+            threshold_info = []
+
+            for id_zone, zone_folder in enumerate(zones_folder):
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                current_counter_index = int(start_index_image)
+                end_counter_index = int(end_index_image)
+
+                # get threshold information
+                path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+                # open treshold path and get this information
+                with open(path_seuil, "r") as seuil_file:
+                    seuil_learned = int(seuil_file.readline().strip())
+
+                threshold_image_found = False
+                while(current_counter_index <= end_counter_index and not threshold_image_found):
+
+                    if seuil_learned < int(current_counter_index):
+                        current_counter_index_str = str(current_counter_index)
+
+                        while len(start_index_image) > len(current_counter_index_str):
+                            current_counter_index_str = "0" + current_counter_index_str
+
+                        threshold_image_found = True
+                        threshold_image_zone = current_counter_index_str
+                        threshold_info.append(threshold_image_zone)
+
+                    current_counter_index += step_counter
+
+                # all indexes of picture to plot
+                images_indexes = [start_index_image, threshold_image_zone, end_index_image]
+                images_data = []
+
+                print(images_indexes)
+
+                for index in images_indexes:
+
+                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
+
+                    current_img = Image.open(img_path)
+                    img_blocks = image_processing.divide_in_blocks(current_img, (200, 200))
+
+                    # getting expected block id
+                    block = img_blocks[id_zone]
+
+                    # get data from mode
+                    # Here you can add the way you compute data
+                    low_bits_block = image_processing.rgb_to_LAB_L_bits(block, (p_shifted + 1, p_shifted + p_bits + 1))
+                    data = metrics.get_SVD_s(low_bits_block)
+
+                    ##################
+                    # Data mode part #
+                    ##################
+
+                    # modify data depending mode
+                    data = image_processing.normalize_arr(data)
+                    images_data.append(data)
+
+                zones_images_data.append(images_data)
+
+            fig=plt.figure(figsize=(8, 8))
+            fig.suptitle('Lab SVD ' + str(p_bits) + ' bits shifted by ' + str(p_shifted) + " for " + p_scene + " scene", fontsize=20)
+
+            for id, data in enumerate(zones_images_data):
+                fig.add_subplot(4, 4, (id + 1))
+                plt.plot(data[0], label='Noisy_' + start_index_image)
+                plt.plot(data[1], label='Threshold_' + threshold_info[id])
+                plt.plot(data[2], label='Reference_' + end_index_image)
+                plt.ylabel('Lab SVD ' + str(p_bits) + ' bits shifted by ' + str(p_shifted) + ', ZONE_' + str(id + 1), fontsize=14)
+                plt.xlabel('Vector features', fontsize=16)
+                plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)
+                plt.ylim(0, 0.1)
+            plt.show()
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python generate_all_data.py --scene A --bits 3 --shifted 3')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hs:b:s", ["help=", "scene=", "bits=", "shifted="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python generate_all_data.py --scene A --bits 3 --shifted 3')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python generate_all_data.py --scene A --bits 3 --shifted 3')
+            sys.exit()
+        elif o in ("-b", "--bits"):
+            p_bits = int(a)
+
+        elif o in ("-s", "--scene"):
+            p_scene = a
+
+            if p_scene not in scenes_indexes:
+                assert False, "Invalid metric choice"
+            else:
+                p_scene = scenes_list[scenes_indexes.index(p_scene)]
+        elif o in ("-f", "--shifted"):
+            p_shifted = int(a)
+        else:
+            assert False, "unhandled option"
+
+    if p_bits + p_shifted > max_nb_bits:
+        assert False, "Invalid parameters, cannot have bits greater than 8 after shift move"
+
+    display_data_scenes(p_scene, p_bits, p_shifted)
+
+if __name__== "__main__":
+    main()

display_simulation_curves.py

@@ -0,0 +1,89 @@
+import numpy as np
+import pandas as pd
+
+import matplotlib.pyplot as plt
+import os, sys, getopt
+
+from modules.utils.data_type import get_svd_data
+
+label_freq = 6
+folder_path = "Curve_simulations"
+
+
+def display_curves(folder_path):
+
+    data_files = os.listdir(folder_path)
+
+    scene_names = [f.split('_')[3] for f in data_files]
+
+    for id, f in enumerate(data_files):
+
+        print(scene_names[id])
+        path_file = os.path.join(folder_path, f)
+
+        df = pd.read_csv(path_file, header=None, sep=";")
+
+
+        fig=plt.figure(figsize=(8, 8))
+        fig.suptitle("Detection simulation for " + scene_names[id] + " scene", fontsize=20)
+
+        for index, row in df.iterrows():
+
+            row = np.asarray(row)
+
+            threshold = row[2]
+            start_index = row[3]
+            step_value = row[4]
+
+            counter_index = 0
+
+            current_value = start_index
+
+            while(current_value < threshold):
+                counter_index += 1
+                current_value += step_value
+
+            fig.add_subplot(4, 4, (index + 1))
+            plt.plot(row[5:])
+
+            # draw vertical line from (70,100) to (70, 250)
+            plt.plot([counter_index, counter_index], [-2, 2], 'k-', lw=2, color='red')
+            plt.ylabel('Not noisy / Noisy', fontsize=18)
+            plt.xlabel('Time in minutes / Samples per pixel', fontsize=16)
+
+            x_labels = [id * step_value + start_index for id, val in enumerate(row[5:]) if id % label_freq == 0]
+
+            x = [v for v in np.arange(0, len(row[5:])+1) if v % label_freq == 0]
+
+            plt.xticks(x, x_labels, rotation=45)
+            plt.ylim(-1, 2)
+
+        plt.show()
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python display_simulation_curves.py --folder "path"')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hm:s:k", ["help=", "folder="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python display_simulation_curves.py --folder "path"')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python display_simulation_curves.py --folder "path"')
+            sys.exit()
+        elif o in ("-f", "--folder"):
+            p_folder = a
+
+        else:
+            assert False, "unhandled option"
+
+
+    display_curves(p_folder)
+
+if __name__== "__main__":
+    main()

display_svd_zone_scene.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep 14 21:02:42 2018
+
+@author: jbuisine
+"""
+
+from __future__ import print_function
+import sys, os, getopt
+
+import numpy as np
+import random
+import time
+import json
+
+from PIL import Image
+from ipfml import image_processing
+from ipfml import metrics
+from skimage import color
+
+import matplotlib.pyplot as plt
+from modules.utils.data_type import get_svd_data
+
+config_filename   = "config"
+zone_folder       = "zone"
+min_max_filename  = "_min_max_values"
+
+# define all scenes values
+scenes_list = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
+metric_choices = ['lab', 'mscn', 'mscn_revisited', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2']
+scenes_indexes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
+choices = ['svd', 'svdn', 'svdne']
+path = './fichiersSVD_light'
+zones = np.arange(16)
+seuil_expe_filename = 'seuilExpe'
+
+max_nb_bits = 8
+
+def display_svd_values(p_scene, p_interval, p_zone, p_metric, p_mode):
+    """
+    @brief Method which gives information about svd curves from zone of picture
+    @param p_scene, scene expected to show svd values
+    @param p_interval, interval [begin, end] of samples or minutes from render generation engine
+    @param p_zone, zone's identifier of picture
+    @param p_metric, metric computed to show
+    @param p_mode, normalization's mode
+    @return nothing
+    """
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    begin, end = p_interval
+    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(scenes):
+
+        if p_scene == folder_scene:
+            print(folder_scene)
+            scene_path = os.path.join(path, folder_scene)
+
+            config_file_path = os.path.join(scene_path, config_filename)
+
+            with open(config_file_path, "r") as config_file:
+                last_image_name = config_file.readline().strip()
+                prefix_image_name = config_file.readline().strip()
+                start_index_image = config_file.readline().strip()
+                end_index_image = config_file.readline().strip()
+                step_counter = int(config_file.readline().strip())
+
+            # construct each zones folder name
+            zones_folder = []
+
+            # get zones list info
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+
+                current_zone = "zone"+index_str
+                zones_folder.append(current_zone)
+
+            zones_images_data = []
+            images_indexes = []
+
+            zone_folder = zones_folder[p_zone]
+
+            zone_path = os.path.join(scene_path, zone_folder)
+
+            current_counter_index = int(start_index_image)
+            end_counter_index = int(end_index_image)
+
+            # get threshold information
+            path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+            # open treshold path and get this information
+            with open(path_seuil, "r") as seuil_file:
+                seuil_learned = int(seuil_file.readline().strip())
+
+            threshold_image_found = False
+
+            while(current_counter_index <= end_counter_index):
+
+                current_counter_index_str = str(current_counter_index)
+
+                while len(start_index_image) > len(current_counter_index_str):
+                    current_counter_index_str = "0" + current_counter_index_str
+
+
+                if current_counter_index >= begin and current_counter_index <= end:
+                    images_indexes.append(current_counter_index_str)
+
+                if seuil_learned < int(current_counter_index) and not threshold_image_found:
+
+                    threshold_image_found = True
+                    threshold_image_zone = current_counter_index_str
+
+                current_counter_index += step_counter
+
+            # all indexes of picture to plot
+            print(images_indexes)
+
+            for index in images_indexes:
+
+                img_path = os.path.join(scene_path, prefix_image_name + str(index) + ".png")
+
+                current_img = Image.open(img_path)
+                img_blocks = image_processing.divide_in_blocks(current_img, (200, 200))
+
+                # getting expected block id
+                block = img_blocks[p_zone]
+
+                # get data from mode
+                # Here you can add the way you compute data
+                data = get_svd_data(p_metric, block)
+
+                ##################
+                # Data mode part #
+                ##################
+
+                if p_mode == 'svdne':
+
+                    # getting max and min information from min_max_filename
+                    with open(data_min_max_filename, 'r') as f:
+                        min_val = float(f.readline())
+                        max_val = float(f.readline())
+
+                    data = image_processing.normalize_arr_with_range(data, min_val, max_val)
+
+                if p_mode == 'svdn':
+                    data = image_processing.normalize_arr(data)
+
+                zones_images_data.append(data)
+
+            plt.title(p_scene + ' scene interval information ['+ str(begin) +', '+ str(end) +'], ' + p_metric + ' metric, ' + p_mode, fontsize=20)
+            plt.ylabel('Image samples or time (minutes) generation', fontsize=14)
+            plt.xlabel('Vector features', fontsize=16)
+
+            for id, data in enumerate(zones_images_data):
+
+                p_label = p_scene + "_" + images_indexes[id]
+
+                if images_indexes[id] == threshold_image_zone:
+                    plt.plot(data, label=p_label, lw=4, color='red')
+                else:
+                    plt.plot(data, label=p_label)
+
+            plt.legend(bbox_to_anchor=(0.8, 1), loc=2, borderaxespad=0.2, fontsize=14)
+            plt.ylim(0, 0.1)
+
+            plt.show()
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python display_svd_zone_scene.py --scene A --interval "0,200" --zone 3 --metric lab --mode svdne')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hs:i:z:l:m", ["help=", "scene=", "interval=", "zone=", "metric=", "mode="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python display_svd_zone_scene.py --scene A --interval "0,200" --zone 3 --metric lab --mode svdne')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python display_svd_zone_scene.py --scene A --interval "0,200" --zone 3 --metric lab --mode svdne')
+            sys.exit()
+        elif o in ("-s", "--scene"):
+            p_scene = a
+
+            if p_scene not in scenes_indexes:
+                assert False, "Invalid scene choice"
+            else:
+                p_scene = scenes_list[scenes_indexes.index(p_scene)]
+        elif o in ("-i", "--interval"):
+            p_interval = list(map(int, a.split(',')))
+
+        elif o in ("-z", "--zone"):
+            p_zone = int(a)
+
+        elif o in ("-m", "--metric"):
+            p_metric = a
+
+            if p_metric not in metric_choices:
+                assert False, "Invalid metric choice"
+
+        elif o in ("-m", "--mode"):
+            p_mode = a
+
+            if p_mode not in choices:
+                assert False, "Invalid normalization choice, expected ['svd', 'svdn', 'svdne']"
+
+        else:
+            assert False, "unhandled option"
+
+    display_svd_values(p_scene, p_interval, p_zone, p_metric, p_mode)
+
+if __name__== "__main__":
+    main()

generate_all_data.py

@@ -13,6 +13,7 @@ import random
 import time
 import json
 
+from modules.utils.data_type import get_svd_data
 from PIL import Image
 from ipfml import image_processing
 from ipfml import metrics
@@ -32,7 +33,8 @@ path = './fichiersSVD_light'
 zones = np.arange(16)
 seuil_expe_filename = 'seuilExpe'
 
-metric_choices = ['lab', 'mscn', 'mscn_revisited', 'low_bits_2', 'low_bits_3', 'low_bits_4']
+metric_choices = ['lab', 'mscn', 'mscn_revisited', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2']
+picture_step = 10
 
 def generate_data_svd(data_type, mode):
     """
@@ -46,7 +48,7 @@ def generate_data_svd(data_type, mode):
     scenes = [s for s in scenes if min_max_filename not in s]
 
     # keep in memory min and max data found from data_type
-    min_val_found = 100000000000
+    min_val_found = sys.maxsize
     max_val_found = 0
 
     data_min_max_filename = os.path.join(path, data_type + min_max_filename)
@@ -95,147 +97,108 @@ def generate_data_svd(data_type, mode):
 
         while(current_counter_index <= end_counter_index):
 
-            current_counter_index_str = str(current_counter_index)
+            if current_counter_index % picture_step == 0:
+                current_counter_index_str = str(current_counter_index)
 
-            while len(start_index_image) > len(current_counter_index_str):
-                current_counter_index_str = "0" + current_counter_index_str
+                while len(start_index_image) > len(current_counter_index_str):
+                    current_counter_index_str = "0" + current_counter_index_str
 
-            img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+                img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
 
-            current_img = Image.open(img_path)
-            img_blocks = image_processing.divide_in_blocks(current_img, (200, 200))
+                current_img = Image.open(img_path)
+                img_blocks = image_processing.divide_in_blocks(current_img, (200, 200))
 
-            for id_block, block in enumerate(img_blocks):
+                for id_block, block in enumerate(img_blocks):
 
-                ###########################
-                # Metric computation part #
-                ###########################
+                    ###########################
+                    # Metric computation part #
+                    ###########################
 
-                # get data from mode
-                # Here you can add the way you compute data
-                if data_type == 'lab':
+                    data = get_svd_data(data_type, block)
 
-                    block_file_path = '/tmp/lab_img.png'
-                    block.save(block_file_path)
-                    data = image_processing.get_LAB_L_SVD_s(Image.open(block_file_path))
+                    ##################
+                    # Data mode part #
+                    ##################
 
-                if data_type == 'mscn_revisited':
+                    # modify data depending mode
+                    if mode == 'svdne':
 
-                    img_mscn_revisited = image_processing.rgb_to_mscn(block)
+                        # getting max and min information from min_max_filename
+                        with open(data_min_max_filename, 'r') as f:
+                            min_val = float(f.readline())
+                            max_val = float(f.readline())
 
-                    # save tmp as img
-                    img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
-                    mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
-                    img_output.save(mscn_revisited_file_path)
-                    img_block = Image.open(mscn_revisited_file_path)
+                        data = image_processing.normalize_arr_with_range(data, min_val, max_val)
 
-                    # extract from temp image
-                    data = metrics.get_SVD_s(img_block)
+                    if mode == 'svdn':
+                        data = image_processing.normalize_arr(data)
 
-                if data_type == 'mscn':
+                    # save min and max found from dataset in order to normalize data using whole data known
+                    if mode == 'svd':
 
-                    img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
-                    img_mscn = image_processing.calculate_mscn_coefficients(img_gray, 7)
-                    img_mscn_norm = image_processing.normalize_2D_arr(img_mscn)
+                        current_min = data.min()
+                        current_max = data.max()
 
-                    img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
+                        if current_min < min_val_found:
+                            min_val_found = current_min
 
-                    data = metrics.get_SVD_s(img_mscn_gray)
+                        if current_max > max_val_found:
+                            max_val_found = current_max
 
-                if data_type == 'low_bits_4':
+                    # now write data into current writer
+                    current_file = svd_output_files[id_block]
 
-                    low_bits_4 = image_processing.rgb_to_LAB_L_low_bits(block)
+                    # add of index
+                    current_file.write(current_counter_index_str + ';')
 
-                    # extract from temp image
-                    data = metrics.get_SVD_s(low_bits_4)
+                    for val in data:
+                        current_file.write(str(val) + ";")
 
-                if data_type == 'low_bits_3':
-
-                    low_bits_3 = image_processing.rgb_to_LAB_L_low_bits(block, 7)
-
-                    # extract from temp image
-                    data = metrics.get_SVD_s(low_bits_3)
-
-                if data_type == 'low_bits_2':
-
-                    low_bits_2 = image_processing.rgb_to_LAB_L_low_bits(block, 3)
-
-                    # extract from temp image
-                    data = metrics.get_SVD_s(low_bits_2)
-
-                ##################
-                # Data mode part #
-                ##################
-
-                # modify data depending mode
-                if mode == 'svdne':
-
-                    # getting max and min information from min_max_filename
-                    with open(data_min_max_filename, 'r') as f:
-                        min_val = float(f.readline())
-                        max_val = float(f.readline())
-
-                    data = image_processing.normalize_arr_with_range(data, min_val, max_val)
-
-                if mode == 'svdn':
-                    data = image_processing.normalize_arr(data)
-
-                # save min and max found from dataset in order to normalize data using whole data known
-                if mode == 'svd':
-
-                    current_min = data.min()
-                    current_max = data.max()
-
-                    if current_min < min_val_found:
-                        min_val_found = current_min
-
-                    if current_max > max_val_found:
-                        max_val_found = current_max
-
-                # now write data into current writer
-                current_file = svd_output_files[id_block]
-
-                # add of index
-                current_file.write(current_counter_index_str + ';')
-
-                for val in data:
-                    current_file.write(str(val) + ";")
-
-                current_file.write('\n')
+                    current_file.write('\n')
 
             start_index_image_int = int(start_index_image)
             print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((current_counter_index - start_index_image_int) / (end_counter_index - start_index_image_int)* 100.) + "%")
+            sys.stdout.write("\033[F")
+
             current_counter_index += step_counter
 
         for f in svd_output_files:
             f.close()
 
+        print('\n')
+
     # save current information about min file found
     if mode == 'svd':
         with open(data_min_max_filename, 'w') as f:
             f.write(str(min_val_found) + '\n')
             f.write(str(max_val_found) + '\n')
 
-    print("End of data generation")
+    print("%s : end of data generation\n" % mode)
 
 
 def main():
 
+    # default value of p_step
+    p_step = 10
+
     if len(sys.argv) <= 1:
         print('Run with default parameters...')
         print('python generate_all_data.py --metric all')
         print('python generate_all_data.py --metric lab')
+        print('python generate_all_data.py --metric lab --step 10')
         sys.exit(2)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "hm", ["help=", "metric="])
+        opts, args = getopt.getopt(sys.argv[1:], "hms", ["help=", "metric=", "step="])
     except getopt.GetoptError:
         # print help information and exit:
-        print('python generate_all_data.py --metric all')
+        print('python generate_all_data.py --metric all --step 10')
         sys.exit(2)
     for o, a in opts:
         if o == "-h":
-            print('python generate_all_data.py --metric all')
+            print('python generate_all_data.py --metric all --step 10')
             sys.exit()
+        elif o in ("-s", "--step"):
+            p_step = int(a)
         elif o in ("-m", "--metric"):
             p_metric = a
 
@@ -244,6 +207,12 @@ def main():
         else:
             assert False, "unhandled option"
 
+    global picture_step
+    picture_step = p_step
+
+    if picture_step % 10 != 0:
+        assert False, "Picture step variable needs to be divided by ten"
+
     # generate all or specific metric data
     if p_metric == 'all':
         for m in metric_choices:

generate_data_model_random.py

@@ -71,7 +71,7 @@ def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes
     test_file = open(output_test_filename, 'w')
 
     scenes = os.listdir(path)
-    
+
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
 
@@ -111,7 +111,7 @@ def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes
                 line = construct_new_line(path_seuil, _interval, lines[index], _sep, _index)
 
                 percent = counter / num_lines
-                
+
                 if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
                     train_file.write(line)
                 else:

generate_data_model_random_maxwell.py

@@ -14,8 +14,7 @@ import time
 import json
 
 from PIL import Image
-from ipfml import image_processing
-from ipfml import metrics
+from ipfml import image_processing, metrics
 
 config_filename   = "config"
 zone_folder       = "zone"
@@ -31,13 +30,21 @@ path = './fichiersSVD_light'
 zones = np.arange(16)
 seuil_expe_filename = 'seuilExpe'
 
-def construct_new_line(path_seuil, interval, line, sep, index):
+min_value_interval = sys.maxsize
+max_value_interval = 0
+
+def construct_new_line(path_seuil, interval, line, norm, sep, index):
     begin, end = interval
 
     line_data = line.split(';')
     seuil = line_data[0]
     metrics = line_data[begin+1:end+1]
 
+    metrics = [float(m) for m in metrics]
+
+    if norm:
+        metrics = image_processing.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
+
     with open(path_seuil, "r") as seuil_file:
         seuil_learned = int(seuil_file.readline().strip())
 
@@ -50,12 +57,71 @@ def construct_new_line(path_seuil, interval, line, sep, index):
         if index:
             line += " " + str(idx + 1)
         line += sep
-        line += val
+        line += str(val)
     line += '\n'
 
     return line
 
-def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes_list, _nb_zones = 4, _percent = 1, _sep=':', _index=True):
+def get_min_max_value_interval(_filename, _interval, _choice, _metric, _scenes = scenes_list, _nb_zones = 4, _percent = 1):
+
+    global min_value_interval, max_value_interval
+
+    scenes = os.listdir(path)
+
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    for id_scene, folder_scene in enumerate(scenes):
+
+        # only take care of maxwell scenes
+        if folder_scene in scenes_list:
+
+            scene_path = os.path.join(path, folder_scene)
+
+            zones_folder = []
+            # create zones list
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+                zones_folder.append("zone"+index_str)
+
+            # shuffle list of zones (=> randomly choose zones)
+            random.shuffle(zones_folder)
+
+            for id_zone, zone_folder in enumerate(zones_folder):
+                zone_path = os.path.join(scene_path, zone_folder)
+                data_filename = _metric + "_" + _choice + generic_output_file_svd
+                data_file_path = os.path.join(zone_path, data_filename)
+
+                # getting number of line and read randomly lines
+                f = open(data_file_path)
+                lines = f.readlines()
+
+                counter = 0
+                # check if user select current scene and zone to be part of training data set
+                for line in lines:
+
+
+                    begin, end = _interval
+
+                    line_data = line.split(';')
+                    metrics = line_data[begin+1:end+1]
+                    metrics = [float(m) for m in metrics]
+
+                    min_value = min(metrics)
+                    max_value = max(metrics)
+
+                    if min_value < min_value_interval:
+                        min_value_interval = min_value
+
+                    if max_value > max_value_interval:
+                        max_value_interval = max_value
+
+                    counter += 1
+
+
+def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes_list, _nb_zones = 4, _percent = 1, _norm = False, _sep=':', _index=True):
 
     output_train_filename = _filename + ".train"
     output_test_filename = _filename + ".test"
@@ -71,7 +137,7 @@ def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes
     test_file = open(output_test_filename, 'w')
 
     scenes = os.listdir(path)
-    
+
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
 
@@ -112,10 +178,10 @@ def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes
                 counter = 0
                 # check if user select current scene and zone to be part of training data set
                 for index in lines_indexes:
-                    line = construct_new_line(path_seuil, _interval, lines[index], _sep, _index)
+                    line = construct_new_line(path_seuil, _interval, lines[index], _norm, _sep, _index)
 
                     percent = counter / num_lines
-                    
+
                     if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
                         train_file.write(line)
                     else:
@@ -133,17 +199,17 @@ def main():
 
     if len(sys.argv) <= 1:
         print('Run with default parameters...')
-        print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1')
+        print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --norm 1 --sep : --rowindex 1')
         sys.exit(2)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:i:k:s:n:p:r", ["help=", "output=", "interval=", "kind=", "metric=","scenes=", "nb_zones=", "percent=", "sep=", "rowindex="])
+        opts, args = getopt.getopt(sys.argv[1:], "ho:i:k:s:n:p:r", ["help=", "output=", "interval=", "kind=", "metric=","scenes=", "nb_zones=", "percent=", "norm=", "sep=", "rowindex="])
     except getopt.GetoptError:
         # print help information and exit:
-        print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1')
+        print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --norm 1 --sep : --rowindex 1')
         sys.exit(2)
     for o, a in opts:
         if o == "-h":
-            print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1')
+            print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --norm 1 --sep : --rowindex 1')
             sys.exit()
         elif o in ("-o", "--output"):
             p_filename = a
@@ -157,6 +223,11 @@ def main():
             p_scenes = a.split(',')
         elif o in ("-n", "--nb_zones"):
             p_nb_zones = int(a)
+        elif o in ("-n", "--norm"):
+            if int(a) == 1:
+                p_norm = True
+            else:
+                p_norm = False
         elif o in ("-p", "--percent"):
             p_percent = float(a)
         elif o in ("-s", "--sep"):
@@ -176,8 +247,12 @@ def main():
         index = scenes_indexes.index(scene_id.strip())
         scenes_selected.append(scenes_list[index])
 
+    # find min max value if necessary to renormalize data
+    if p_norm:
+        get_min_max_value_interval(p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent)
+
     # create database using img folder (generate first time only)
-    generate_data_model(p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_sep, p_rowindex)
+    generate_data_model(p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_norm, p_sep, p_rowindex)
 
 if __name__== "__main__":
     main()

metrics_predictions/predict_noisy_image_svd_low_bits_4_shifted_2.py

@@ -0,0 +1,78 @@
+from sklearn.externals import joblib
+
+import numpy as np
+
+from ipfml import image_processing
+from ipfml import metrics
+from PIL import Image
+
+import sys, os, getopt
+
+min_max_file_path = 'fichiersSVD_light/low_bits_4_shifted_2_min_max_values'
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "0,20" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hi:t:m:o", ["help=", "image=", "interval=", "model=", "mode="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+            sys.exit()
+        elif o in ("-i", "--image"):
+            p_img_file = os.path.join(os.path.join(os.path.dirname(__file__),'../'), a)
+        elif o in ("-t", "--interval"):
+            p_interval = list(map(int, a.split(',')))
+        elif o in ("-m", "--model"):
+            p_model_file = os.path.join(os.path.join(os.path.dirname(__file__),'../'), a)
+        elif o in ("-o", "--mode"):
+            p_mode = a
+
+            if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
+                assert False, "Mode not recognized"
+        else:
+            assert False, "unhandled option"
+
+    # load of model file
+    model = joblib.load(p_model_file)
+
+    # load image
+    img = Image.open(p_img_file)
+    low_bits_4_shifted_2_values = metrics.get_SVD_s(image_processing.rgb_to_LAB_L_bits(block_used, (3, 6)))
+
+    # check mode to normalize data
+    if p_mode == 'svdne':
+
+        # need to read min_max_file
+        file_path = os.path.join(os.path.join(os.path.dirname(__file__),'../'), min_max_file_path)
+        with open(file_path, 'r') as f:
+            min = float(f.readline().replace('\n', ''))
+            max = float(f.readline().replace('\n', ''))
+
+        l_values = image_processing.normalize_arr_with_range(low_bits_4_shifted_2_values, min, max)
+
+    elif p_mode == 'svdn':
+        l_values = image_processing.normalize_arr(low_bits_4_shifted_2_values)
+    else:
+        l_values = low_bits_4_shifted_2_values
+
+
+    # get interval values
+    begin, end = p_interval
+    test_data = l_values[begin:end]
+
+    # get prediction of model
+    prediction = model.predict([test_data])[0]
+
+    print(prediction)
+
+
+if __name__== "__main__":
+    main()

metrics_predictions/predict_noisy_image_svd_low_bits_5.py

@@ -0,0 +1,78 @@
+from sklearn.externals import joblib
+
+from ipfml import image_processing
+from ipfml import metrics
+
+from PIL import Image
+
+import sys, os, getopt
+import numpy as np
+
+min_max_file_path = 'fichiersSVD_light/low_bits_5_min_max_values'
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "0,20" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hi:t:m:o", ["help=", "image=", "interval=", "model=", "mode="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+            sys.exit()
+        elif o in ("-i", "--image"):
+            p_img_file =  os.path.join(os.path.join(os.path.dirname(__file__),'../'), a)
+        elif o in ("-t", "--interval"):
+            p_interval = list(map(int, a.split(',')))
+        elif o in ("-m", "--model"):
+            p_model_file = os.path.join(os.path.join(os.path.dirname(__file__),'../'), a)
+        elif o in ("-o", "--mode"):
+            p_mode = a
+
+            if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
+                assert False, "Mode not recognized"
+        else:
+            assert False, "unhandled option"
+
+    # load of model file
+    model = joblib.load(p_model_file)
+
+    # load image
+    img = Image.open(p_img_file)
+    low_bits_5_values = metrics.get_SVD_s(image_processing.rgb_to_LAB_L_low_bits(img, 31))
+
+    # check mode to normalize data
+    if p_mode == 'svdne':
+
+        # need to read min_max_file
+        file_path = os.path.join(os.path.join(os.path.dirname(__file__),'../'), min_max_file_path)
+        with open(file_path, 'r') as f:
+            min = float(f.readline().replace('\n', ''))
+            max = float(f.readline().replace('\n', ''))
+
+        l_values = image_processing.normalize_arr_with_range(low_bits_5_values, min, max)
+
+    elif p_mode == 'svdn':
+        l_values = image_processing.normalize_arr(low_bits_5_values)
+    else:
+        l_values = low_bits_6_values
+
+
+    # get interval values
+    begin, end = p_interval
+    test_data = l_values[begin:end]
+
+    # get prediction of model
+    prediction = model.predict([test_data])[0]
+
+    print(prediction)
+
+
+if __name__== "__main__":
+    main()

metrics_predictions/predict_noisy_image_svd_low_bits_6.py

@@ -0,0 +1,78 @@
+from sklearn.externals import joblib
+
+from ipfml import image_processing
+from ipfml import metrics
+
+from PIL import Image
+
+import sys, os, getopt
+import numpy as np
+
+min_max_file_path = 'fichiersSVD_light/low_bits_6_min_max_values'
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "0,20" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hi:t:m:o", ["help=", "image=", "interval=", "model=", "mode="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python predict_noisy_image_svd_lab.py --image path/to/xxxx --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svdn", "svdne"]')
+            sys.exit()
+        elif o in ("-i", "--image"):
+            p_img_file =  os.path.join(os.path.join(os.path.dirname(__file__),'../'), a)
+        elif o in ("-t", "--interval"):
+            p_interval = list(map(int, a.split(',')))
+        elif o in ("-m", "--model"):
+            p_model_file = os.path.join(os.path.join(os.path.dirname(__file__),'../'), a)
+        elif o in ("-o", "--mode"):
+            p_mode = a
+
+            if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
+                assert False, "Mode not recognized"
+        else:
+            assert False, "unhandled option"
+
+    # load of model file
+    model = joblib.load(p_model_file)
+
+    # load image
+    img = Image.open(p_img_file)
+    low_bits_6_values = metrics.get_SVD_s(image_processing.rgb_to_LAB_L_low_bits(img, 63))
+
+    # check mode to normalize data
+    if p_mode == 'svdne':
+
+        # need to read min_max_file
+        file_path = os.path.join(os.path.join(os.path.dirname(__file__),'../'), min_max_file_path)
+        with open(file_path, 'r') as f:
+            min = float(f.readline().replace('\n', ''))
+            max = float(f.readline().replace('\n', ''))
+
+        l_values = image_processing.normalize_arr_with_range(low_bits_6_values, min, max)
+
+    elif p_mode == 'svdn':
+        l_values = image_processing.normalize_arr(low_bits_6_values)
+    else:
+        l_values = low_bits_6_values
+
+
+    # get interval values
+    begin, end = p_interval
+    test_data = l_values[begin:end]
+
+    # get prediction of model
+    prediction = model.predict([test_data])[0]
+
+    print(prediction)
+
+
+if __name__== "__main__":
+    main()

modules/utils/data_type.py

@@ -0,0 +1,83 @@
+from ipfml import image_processing, metrics
+from PIL import Image
+from skimage import color
+
+import numpy as np
+
+def get_svd_data(data_type, block):
+    """
+    Method which returns the data type expected
+    """
+
+    if data_type == 'lab':
+
+        block_file_path = '/tmp/lab_img.png'
+        block.save(block_file_path)
+        data = image_processing.get_LAB_L_SVD_s(Image.open(block_file_path))
+
+    if data_type == 'mscn_revisited':
+
+        img_mscn_revisited = image_processing.rgb_to_mscn(block)
+
+        # save tmp as img
+        img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
+        mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
+        img_output.save(mscn_revisited_file_path)
+        img_block = Image.open(mscn_revisited_file_path)
+
+        # extract from temp image
+        data = metrics.get_SVD_s(img_block)
+
+    if data_type == 'mscn':
+
+        img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
+        img_mscn = image_processing.calculate_mscn_coefficients(img_gray, 7)
+        img_mscn_norm = image_processing.normalize_2D_arr(img_mscn)
+
+        img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
+
+        data = metrics.get_SVD_s(img_mscn_gray)
+
+    if data_type == 'low_bits_6':
+
+        low_bits_6 = image_processing.rgb_to_LAB_L_low_bits(block, 63)
+
+        # extract from temp image
+        data = metrics.get_SVD_s(low_bits_6)
+
+    if data_type == 'low_bits_5':
+
+        low_bits_5 = image_processing.rgb_to_LAB_L_low_bits(block, 31)
+
+        # extract from temp image
+        data = metrics.get_SVD_s(low_bits_5)
+
+
+    if data_type == 'low_bits_4':
+
+        low_bits_4 = image_processing.rgb_to_LAB_L_low_bits(block)
+
+        # extract from temp image
+        data = metrics.get_SVD_s(low_bits_4)
+
+    if data_type == 'low_bits_3':
+
+        low_bits_3 = image_processing.rgb_to_LAB_L_low_bits(block, 7)
+
+        # extract from temp image
+        data = metrics.get_SVD_s(low_bits_3)
+
+    if data_type == 'low_bits_2':
+
+        low_bits_2 = image_processing.rgb_to_LAB_L_low_bits(block, 3)
+
+        # extract from temp image
+        data = metrics.get_SVD_s(low_bits_2)
+
+    if data_type == 'low_bits_4_shifted_2':
+
+        data = metrics.get_SVD_s(image_processing.rgb_to_LAB_L_bits(block, (3, 6)))
+
+    return data
+
+

predict_seuil_expe_maxwell.py

@@ -49,7 +49,7 @@ def main():
 
             if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
                 assert False, "Mode not recognized"
-    
+
         elif o in ("-m", "--metric"):
             p_metric = a
         elif o in ("-l", "--limit_detection"):
@@ -58,16 +58,16 @@ def main():
             assert False, "unhandled option"
 
     scenes = os.listdir(scenes_path)
-    
+
     if min_max_filename in scenes:
         scenes.remove(min_max_filename)
 
     # go ahead each scenes
     for id_scene, folder_scene in enumerate(scenes):
-    
+
         # only take in consideration maxwell scenes
         if folder_scene in maxwell_scenes:
-    
+
             print(folder_scene)
 
             scene_path = os.path.join(scenes_path, folder_scene)
@@ -111,7 +111,7 @@ def main():
             check_all_done = False
 
             while(current_counter_index <= end_counter_index and not check_all_done):
-                
+
                 current_counter_index_str = str(current_counter_index)
 
                 while len(start_index_image) > len(current_counter_index_str):
@@ -126,7 +126,7 @@ def main():
                 check_all_done = all(d == True for d in threshold_expes_detected)
 
                 for id_block, block in enumerate(img_blocks):
-                    
+
                     # check only if necessary for this scene (not already detected)
                     if not threshold_expes_detected[id_block]:
 
@@ -140,9 +140,9 @@ def main():
 
                         ## call command ##
                         p = subprocess.Popen(python_cmd, stdout=subprocess.PIPE, shell=True)
-                        
+
                         (output, err) = p.communicate()
-                        
+
                         ## Wait for result ##
                         p_status = p.wait()
 
@@ -152,7 +152,7 @@ def main():
                             threshold_expes_counter[id_block] = threshold_expes_counter[id_block] + 1
                         else:
                             threshold_expes_counter[id_block] = 0
-                        
+
                         if threshold_expes_counter[id_block] == p_limit:
                             threshold_expes_detected[id_block] = True
                             threshold_expes_found[id_block] = current_counter_index
@@ -161,14 +161,14 @@ def main():
 
                 current_counter_index += step_counter
                 print("------------------------")
-                print("Scene " + str(id_scene + 1) + "/" + str(len(scenes)))
+                print("Scene " + str(id_scene + 1) + "/" + str(len(maxwell_scenes)))
                 print("------------------------")
 
             # end of scene => display of results
 
             # construct path using model name for saving threshold map folder
             model_treshold_path = os.path.join(threshold_map_folder, p_model_file.split('/')[-1].replace('.joblib', ''))
-            
+
             # create threshold model path if necessary
             if not os.path.exists(model_treshold_path):
                 os.makedirs(model_treshold_path)
@@ -191,7 +191,7 @@ def main():
                 if (id + 1) % 4 == 0:
                     f_map.write(line_information + '\n')
                     line_information = ""
-            
+
             f_map.write(line_information + '\n')
 
             min_abs_dist = min(abs_dist)
@@ -220,4 +220,4 @@ def main():
 
 
 if __name__== "__main__":
-    main()
+    main()

predict_seuil_expe_maxwell_curve.py

@@ -0,0 +1,180 @@
+from sklearn.externals import joblib
+
+import numpy as np
+
+from ipfml import image_processing
+from PIL import Image
+
+import sys, os, getopt
+import subprocess
+import time
+
+current_dirpath = os.getcwd()
+
+config_filename   = "config"
+scenes_path = './fichiersSVD_light'
+min_max_filename = '_min_max_values'
+threshold_expe_filename = 'seuilExpe'
+tmp_filename = '/tmp/__model__img_to_predict.png'
+
+maxwell_scenes = ['Appart1opt02', 'Cuisine01', 'SdbCentre', 'SdbDroite']
+
+threshold_map_folder = "threshold_map"
+simulation_curves_zones = "simulation_curves_zones_"
+
+zones = np.arange(16)
+
+def main():
+
+    if len(sys.argv) <= 1:
+        print('Run with default parameters...')
+        print('python predict_seuil_expe_maxwell.py --interval "0,20" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "ht:m:o:l", ["help=", "interval=", "model=", "mode=", "metric=", "limit_detection="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('python predict_seuil_expe_maxwell.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('python predict_seuil_expe_maxwell.py --interval "xx,xx" --model path/to/xxxx.joblib --mode svdn --metric lab --limit_detection xx')
+            sys.exit()
+        elif o in ("-t", "--interval"):
+            p_interval = a
+        elif o in ("-m", "--model"):
+            p_model_file = a
+        elif o in ("-o", "--mode"):
+            p_mode = a
+
+            if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
+                assert False, "Mode not recognized"
+
+        elif o in ("-m", "--metric"):
+            p_metric = a
+        elif o in ("-l", "--limit_detection"):
+            p_limit = int(a)
+        else:
+            assert False, "unhandled option"
+
+    scenes = os.listdir(scenes_path)
+
+    if min_max_filename in scenes:
+        scenes.remove(min_max_filename)
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(scenes):
+
+        # only take in consideration maxwell scenes
+        if folder_scene in maxwell_scenes:
+
+            print(folder_scene)
+
+            scene_path = os.path.join(scenes_path, folder_scene)
+
+            config_path = os.path.join(scene_path, config_filename)
+
+            with open(config_path, "r") as config_file:
+                last_image_name = config_file.readline().strip()
+                prefix_image_name = config_file.readline().strip()
+                start_index_image = config_file.readline().strip()
+                end_index_image = config_file.readline().strip()
+                step_counter = int(config_file.readline().strip())
+
+            threshold_expes = []
+            threshold_expes_found = []
+            block_predictions_str = []
+
+            # get zones list info
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+                zone_folder = "zone"+index_str
+
+                threshold_path_file = os.path.join(os.path.join(scene_path, zone_folder), threshold_expe_filename)
+
+                with open(threshold_path_file) as f:
+                    threshold = int(f.readline())
+                    threshold_expes.append(threshold)
+
+                    # Initialize default data to get detected model threshold found
+                    threshold_expes_found.append(int(end_index_image)) # by default use max
+
+                block_predictions_str.append(index_str + ";" + p_model_file + ";" + str(threshold) + ";" + str(start_index_image) + ";" + str(step_counter))
+
+            current_counter_index = int(start_index_image)
+            end_counter_index = int(end_index_image)
+
+            print(current_counter_index)
+
+            while(current_counter_index <= end_counter_index):
+
+                current_counter_index_str = str(current_counter_index)
+
+                while len(start_index_image) > len(current_counter_index_str):
+                    current_counter_index_str = "0" + current_counter_index_str
+
+                img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+
+                current_img = Image.open(img_path)
+                img_blocks = image_processing.divide_in_blocks(current_img, (200, 200))
+
+                for id_block, block in enumerate(img_blocks):
+
+                    # check only if necessary for this scene (not already detected)
+                    #if not threshold_expes_detected[id_block]:
+
+                        tmp_file_path = tmp_filename.replace('__model__',  p_model_file.split('/')[-1].replace('.joblib', '_'))
+                        block.save(tmp_file_path)
+
+                        python_cmd = "python metrics_predictions/predict_noisy_image_svd_" + p_metric + ".py --image " + tmp_file_path + \
+                                        " --interval '" + p_interval + \
+                                        "' --model " + p_model_file  + \
+                                        " --mode " + p_mode
+
+                        ## call command ##
+                        p = subprocess.Popen(python_cmd, stdout=subprocess.PIPE, shell=True)
+
+                        (output, err) = p.communicate()
+
+                        ## Wait for result ##
+                        p_status = p.wait()
+
+                        prediction = int(output)
+
+                        # save here in specific file of block all the predictions done
+                        block_predictions_str[id_block] = block_predictions_str[id_block] + ";" + str(prediction)
+
+                        print(str(id_block) + " : " + str(current_counter_index) + "/" + str(threshold_expes[id_block]) + " => " + str(prediction))
+
+                current_counter_index += step_counter
+                print("------------------------")
+                print("Scene " + str(id_scene + 1) + "/" + str(len(scenes)))
+                print("------------------------")
+
+            # end of scene => display of results
+
+            # construct path using model name for saving threshold map folder
+            model_treshold_path = os.path.join(threshold_map_folder, p_model_file.split('/')[-1].replace('.joblib', ''))
+
+            # create threshold model path if necessary
+            if not os.path.exists(model_treshold_path):
+                os.makedirs(model_treshold_path)
+
+            map_filename = os.path.join(model_treshold_path, simulation_curves_zones + folder_scene)
+            f_map = open(map_filename, 'w')
+
+            for line in block_predictions_str:
+                f_map.write(line + '\n')
+            f_map.close()
+
+            print("Scene " + str(id_scene + 1) + "/" + str(len(maxwell_scenes)) + " Done..")
+            print("------------------------")
+
+            print("Model predictions are saved into %s" map_filename)
+            time.sleep(10)
+
+
+if __name__== "__main__":
+    main()

runAll_maxwell.sh

@@ -2,22 +2,23 @@
 
 # erase "models_info/models_comparisons.csv" file and write new header
 file_path='models_info/models_comparisons.csv'
-rm ${file_path}
 
 erased=$1
 
-if [ "$erased" == "Y" ]; then
+if [ "${erased}" == "Y" ]; then
     echo "Previous data file erased..."
+    rm ${file_path}
     mkdir -p models_info
     touch ${file_path}
-fi
 
-# add of header
-echo 'model_name; vector_size; start; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; F1_val; F1_test; F1_all' >> ${file_path}
+    # add of header
+    echo 'model_name; vector_size; start; end; nb_zones; metric; mode; tran_size; val_size; test_size; train_pct_size; val_pct_size; test_pct_size; train_acc; val_acc; test_acc; all_acc; F1_train; F1_val; F1_test; F1_all' >> ${file_path}
+
+fi
 
 for size in {"4","8","16","26","32","40"}; do
 
-    for metric in {"lab","mscn","mscn_revisited","low_bits_2","low_bits_3","low_bits_4"}; do
+    for metric in {"lab","mscn","mscn_revisited","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2"}; do
         bash generateAndTrain_maxwell.sh ${size} ${metric}
     done
 done

run_maxwell_simulation.sh

@@ -0,0 +1,63 @@
+#! bin/bash
+
+# file which contains model names we want to use for simulation
+simulate_models="simulate_models.csv"
+
+# selection of four scenes (only maxwell)
+scenes="A, D, G, H"
+VECTOR_SIZE=200
+
+for size in {"4","8","16","26","32","40"}; do
+    for metric in {"lab","mscn","mscn_revisited","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2"}; do
+
+        half=$(($size/2))
+        start=-$half
+
+        for counter in {0..4}; do
+             end=$(($start+$size))
+
+             if [ "$end" -gt "$VECTOR_SIZE" ]; then
+                 start=$(($VECTOR_SIZE-$size))
+                 end=$(($VECTOR_SIZE))
+             fi
+
+             if [ "$start" -lt "0" ]; then
+                 start=$((0))
+                 end=$(($size))
+             fi
+
+             for nb_zones in {4,6,8,10,12,14}; do
+
+                 for mode in {"svd","svdn","svdne"}; do
+                     for model in {"svm_model","ensemble_model","ensemble_model_v2"}; do
+
+                        FILENAME="data/data_maxwell_N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${metric}_${mode}"
+
+                        MODEL_NAME="${model}_N${size}_B${start}_E${end}_nb_zones_${nb_zones}_${metric}_${mode}"
+
+                        if grep -q "${MODEL_NAME}" "${simulate_models}"; then
+                            echo "Run simulation for model ${MODEL_NAME}"
+
+                            # by default regenerate model
+                            python generate_data_model_random_maxwell.py --output ${FILENAME} --interval "${start},${end}" --kind ${mode} --metric ${metric} --scenes "${scenes}" --nb_zones "${nb_zones}" --percent 1 --sep ';' --rowindex '0'
+
+                            python models/${model}_train.py --data ${FILENAME} --output ${MODEL_NAME}
+
+                            python predict_seuil_expe_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric} --limit_detection '2'
+
+                            python save_model_result_in_md_maxwell.py --interval "${start},${end}" --model "saved_models/${MODEL_NAME}.joblib" --mode "${mode}" --metric ${metric}
+
+                        fi
+                    done
+                done
+            done
+
+            if [ "$counter" -eq "0" ]; then
+                start=$(($start+50-$half))
+            else
+                start=$(($start+50))
+            fi
+
+        done
+    done
+done