Refactoring of all display script

.gitignore

@@ -1,11 +1,13 @@
 # project data
-data/*
-saved_models/*
-threshold_map/*
-models_info/*
-custom_norm/*
-learned_zones/*
-corr_indices/*
+data
+saved_models
+threshold_map
+models_info
+custom_norm
+learned_zones
+corr_indices
+results
+metric_curves
 .ipynb_checkpoints
 
 # simulate_models.csv

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "modules"]
+	path = modules
+	url = https://github.com/prise-3d/Thesis-CommonModules.git

README.md

@@ -178,4 +178,4 @@ All others bash scripts are used to combine and run multiple model combinations.
 
 ## License
 
-[The MIT license](https://github.com/prise-3d/Thesis-NoiseDetection-metrics/blob/master/LICENSE)
+[The MIT license](https://github.com/prise-3d/Thesis-NoiseDetection-attributes/blob/master/LICENSE)

custom_config.py

@@ -0,0 +1,19 @@
+from modules.config.attributes_config import *
+
+# store all variables from global config
+context_vars = vars()
+
+# folders
+## min_max_custom_folder           = 'custom_norm'
+## correlation_indices_folder      = 'corr_indices'
+
+# variables
+## features_choices_labels         = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed', 'mscn_var_4', 'mscn_var_16', 'mscn_var_64', 'mscn_var_16_max', 'mscn_var_64_max', 'ica_diff', 'svd_trunc_diff', 'ipca_diff', 'svd_reconstruct', 'highest_sv_std_filters', 'lowest_sv_std_filters', 'highest_wave_sv_std_filters', 'lowest_wave_sv_std_filters']
+
+## models_names_list               = ["svm_model","ensemble_model","ensemble_model_v2","deep_keras"]
+## normalization_choices           = ['svd', 'svdn', 'svdne']
+
+# parameters
+## keras_epochs                    = 500
+## keras_batch                     = 32
+## val_dataset_size                = 0.2

modules/utils/data.py

@@ -1,25 +1,26 @@
-from ipfml import processing, metrics, utils
-from modules.utils.config import *
+# main imports
+import numpy as np
+import sys
 
+# image transform imports
 from PIL import Image
 from skimage import color
 from sklearn.decomposition import FastICA
 from sklearn.decomposition import IncrementalPCA
 from sklearn.decomposition import TruncatedSVD
 from numpy.linalg import svd as lin_svd
-
 from scipy.signal import medfilt2d, wiener, cwt
 import pywt
-
 import cv2
-import numpy as np
 
+from ipfml.processing import transform, compression, segmentation
+from ipfml import utils
 
-_scenes_names_prefix   = '_scenes_names'
-_scenes_indices_prefix = '_scenes_indices'
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-# store all variables from current module context
-context_vars = vars()
+import custom_config as cfg
+from modules.utils import data as dt
 
 
 def get_svd_data(data_type, block):
@@ -31,11 +32,11 @@ def get_svd_data(data_type, block):
 
         block_file_path = '/tmp/lab_img.png'
         block.save(block_file_path)
-        data = processing.get_LAB_L_SVD_s(Image.open(block_file_path))
+        data = transform.get_LAB_L_SVD_s(Image.open(block_file_path))
 
     if data_type == 'mscn':
 
-        img_mscn_revisited = processing.rgb_to_mscn(block)
+        img_mscn_revisited = transform.rgb_to_mscn(block)
 
         # save tmp as img
         img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
@@ -44,47 +45,47 @@ def get_svd_data(data_type, block):
         img_block = Image.open(mscn_revisited_file_path)
 
         # extract from temp image
-        data = metrics.get_SVD_s(img_block)
+        data = compression.get_SVD_s(img_block)
 
     """if data_type == 'mscn':
 
         img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
-        img_mscn = processing.calculate_mscn_coefficients(img_gray, 7)
-        img_mscn_norm = processing.normalize_2D_arr(img_mscn)
+        img_mscn = transform.calculate_mscn_coefficients(img_gray, 7)
+        img_mscn_norm = transform.normalize_2D_arr(img_mscn)
 
         img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
 
-        data = metrics.get_SVD_s(img_mscn_gray)
+        data = compression.get_SVD_s(img_mscn_gray)
     """
 
     if data_type == 'low_bits_6':
 
-        low_bits_6 = processing.rgb_to_LAB_L_low_bits(block, 6)
-        data = metrics.get_SVD_s(low_bits_6)
+        low_bits_6 = transform.rgb_to_LAB_L_low_bits(block, 6)
+        data = compression.get_SVD_s(low_bits_6)
 
     if data_type == 'low_bits_5':
 
-        low_bits_5 = processing.rgb_to_LAB_L_low_bits(block, 5)
-        data = metrics.get_SVD_s(low_bits_5)
+        low_bits_5 = transform.rgb_to_LAB_L_low_bits(block, 5)
+        data = compression.get_SVD_s(low_bits_5)
 
     if data_type == 'low_bits_4':
 
-        low_bits_4 = processing.rgb_to_LAB_L_low_bits(block, 4)
-        data = metrics.get_SVD_s(low_bits_4)
+        low_bits_4 = transform.rgb_to_LAB_L_low_bits(block, 4)
+        data = compression.get_SVD_s(low_bits_4)
 
     if data_type == 'low_bits_3':
 
-        low_bits_3 = processing.rgb_to_LAB_L_low_bits(block, 3)
-        data = metrics.get_SVD_s(low_bits_3)
+        low_bits_3 = transform.rgb_to_LAB_L_low_bits(block, 3)
+        data = compression.get_SVD_s(low_bits_3)
 
     if data_type == 'low_bits_2':
 
-        low_bits_2 = processing.rgb_to_LAB_L_low_bits(block, 2)
-        data = metrics.get_SVD_s(low_bits_2)
+        low_bits_2 = transform.rgb_to_LAB_L_low_bits(block, 2)
+        data = compression.get_SVD_s(low_bits_2)
 
     if data_type == 'low_bits_4_shifted_2':
 
-        data = metrics.get_SVD_s(processing.rgb_to_LAB_L_bits(block, (3, 6)))
+        data = compression.get_SVD_s(transform.rgb_to_LAB_L_bits(block, (3, 6)))
 
     if data_type == 'sub_blocks_stats':
 
@@ -92,14 +93,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         sub_width, sub_height = int(width / 4), int(height / 4)
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
         data = []
 
         for sub_b in sub_blocks:
 
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
 
             # get information we want from svd
             data.append(np.mean(l_svd_data))
@@ -120,14 +121,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         sub_width, sub_height = int(width / 4), int(height / 4)
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
         data = []
 
         for sub_b in sub_blocks:
 
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
 
             # get information we want from svd
             data.append(np.mean(l_svd_data))
@@ -145,14 +146,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         sub_width, sub_height = int(width / 8), int(height / 8)
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
         data = []
 
         for sub_b in sub_blocks:
 
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
 
             area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
             data.append(area_under_curve)
@@ -166,14 +167,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         sub_width, sub_height = int(width / 8), int(height / 8)
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
         data = []
 
         for sub_b in sub_blocks:
 
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
             l_svd_data = utils.normalize_arr(l_svd_data)
 
             area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
@@ -211,7 +212,7 @@ def get_svd_data(data_type, block):
         data = data[indices]
 
     if data_type == 'ica_diff':
-        current_image = metrics.get_LAB_L(block)
+        current_image = transform.get_LAB_L(block)
 
         ica = FastICA(n_components=50)
         ica.fit(current_image)
@@ -222,14 +223,14 @@ def get_svd_data(data_type, block):
         final_image = utils.normalize_2D_arr(image_restored)
         final_image = np.array(final_image * 255, 'uint8')
 
-        sv_values = utils.normalize_arr(metrics.get_SVD_s(current_image))
-        ica_sv_values = utils.normalize_arr(metrics.get_SVD_s(final_image))
+        sv_values = utils.normalize_arr(compression.get_SVD_s(current_image))
+        ica_sv_values = utils.normalize_arr(compression.get_SVD_s(final_image))
 
         data = abs(np.array(sv_values) - np.array(ica_sv_values))
 
     if data_type == 'svd_trunc_diff':
 
-        current_image = metrics.get_LAB_L(block)
+        current_image = transform.get_LAB_L(block)
 
         svd = TruncatedSVD(n_components=30, n_iter=100, random_state=42)
         transformed_image = svd.fit_transform(current_image)
@@ -237,12 +238,12 @@ def get_svd_data(data_type, block):
 
         reduced_image = (current_image - restored_image)
 
-        U, s, V = metrics.get_SVD(reduced_image)
+        U, s, V = compression.get_SVD(reduced_image)
         data = s
 
     if data_type == 'ipca_diff':
 
-        current_image = metrics.get_LAB_L(block)
+        current_image = transform.get_LAB_L(block)
 
         transformer = IncrementalPCA(n_components=20, batch_size=25)
         transformed_image = transformer.fit_transform(current_image)
@@ -250,7 +251,7 @@ def get_svd_data(data_type, block):
 
         reduced_image = (current_image - restored_image)
 
-        U, s, V = metrics.get_SVD(reduced_image)
+        U, s, V = compression.get_SVD(reduced_image)
         data = s
 
     if data_type == 'svd_reconstruct':
@@ -258,7 +259,7 @@ def get_svd_data(data_type, block):
         reconstructed_interval = (90, 200)
         begin, end = reconstructed_interval
 
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         lab_img = np.array(lab_img, 'uint8')
 
         U, s, V = lin_svd(lab_img, full_matrices=True)
@@ -269,12 +270,12 @@ def get_svd_data(data_type, block):
 
         output_img = np.array(output_img, 'uint8')
 
-        data = metrics.get_SVD_s(output_img)
+        data = compression.get_SVD_s(output_img)
 
     if 'sv_std_filters' in data_type:
 
         # convert into lab by default to apply filters
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         arr = np.array(lab_img)
         images = []
         
@@ -285,12 +286,12 @@ def get_svd_data(data_type, block):
         images.append(wiener(arr, [5, 5]))
         
         # By default computation of current block image
-        s_arr = metrics.get_SVD_s(arr)
+        s_arr = compression.get_SVD_s(arr)
         sv_vector = [s_arr]
 
         # for each new image apply SVD and get SV 
         for img in images:
-            s = metrics.get_SVD_s(img)
+            s = compression.get_SVD_s(img)
             sv_vector.append(s)
             
         sv_array = np.array(sv_vector)
@@ -307,10 +308,10 @@ def get_svd_data(data_type, block):
         indices = []
 
         if 'lowest' in data_type:
-            indices = get_lowest_values(sv_std, 200)
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
 
         if 'highest' in data_type:
-            indices = get_highest_values(sv_std, 200)
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
 
         # data are arranged following std trend computed
         data = s_arr[indices]
@@ -319,7 +320,7 @@ def get_svd_data(data_type, block):
     if 'wave_sv_std_filters' in data_type:
 
         # convert into lab by default to apply filters
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         arr = np.array(lab_img)
         images = []
         
@@ -335,12 +336,12 @@ def get_svd_data(data_type, block):
         images.append(w2d(arr, 'haar', 4))
         
         # By default computation of current block image
-        s_arr = metrics.get_SVD_s(arr)
+        s_arr = compression.get_SVD_s(arr)
         sv_vector = [s_arr]
 
         # for each new image apply SVD and get SV 
         for img in images:
-            s = metrics.get_SVD_s(img)
+            s = compression.get_SVD_s(img)
             sv_vector.append(s)
             
         sv_array = np.array(sv_vector)
@@ -357,10 +358,10 @@ def get_svd_data(data_type, block):
         indices = []
 
         if 'lowest' in data_type:
-            indices = get_lowest_values(sv_std, 200)
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
 
         if 'highest' in data_type:
-            indices = get_highest_values(sv_std, 200)
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
 
         # data are arranged following std trend computed
         data = s_arr[indices]
@@ -369,7 +370,7 @@ def get_svd_data(data_type, block):
 
         img_width, img_height = 200, 200
 
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         arr = np.array(lab_img)
 
         # compute all filters statistics
@@ -429,14 +430,6 @@ def get_svd_data(data_type, block):
     return data
 
 
-def get_highest_values(arr, n):
-    return np.array(arr).argsort()[-n:][::-1]
-
-
-def get_lowest_values(arr, n):
-    return np.array(arr).argsort()[::-1][-n:][::-1]
-
-
 def w2d(arr, mode='haar', level=1):
     #convert to float   
     imArray = arr
@@ -458,35 +451,14 @@ def w2d(arr, mode='haar', level=1):
 
 def _get_mscn_variance(block, sub_block_size=(50, 50)):
 
-    blocks = processing.divide_in_blocks(block, sub_block_size)
+    blocks = segmentation.divide_in_blocks(block, sub_block_size)
 
     data = []
 
     for block in blocks:
-        mscn_coefficients = processing.get_mscn_coefficients(block)
+        mscn_coefficients = transform.get_mscn_coefficients(block)
         flat_coeff = mscn_coefficients.flatten()
         data.append(np.var(flat_coeff))
 
     return np.sort(data)
 
-
-def get_renderer_scenes_indices(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_indices
-    else:
-        return context_vars[renderer_name + _scenes_indices_prefix]
-
-def get_renderer_scenes_names(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_names
-    else:
-        return context_vars[renderer_name + _scenes_names_prefix]
-

dataset

@@ -0,0 +1 @@
+../data/Scenes/

display_bits_shifted_scene.py

@@ -1,40 +1,34 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
 import random
 import time
 import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing
-from ipfml import metrics
 from skimage import color
 import matplotlib.pyplot as plt
 
-from modules.utils import config as cfg
+from ipfml.processing import compression, transform
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
+import custom_config as cfg
+from modules.utils import data as dt
 
-config_filename     = cfg.config_filename
+# variables and parameters
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 
 # define all scenes values
 scenes_list         = cfg.scenes_names
 scenes_indices      = cfg.scenes_indices
-choices             = cfg.normalization_choices
 path                = cfg.dataset_path
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 
-metric_choices      = cfg.metric_choices_labels
 max_nb_bits = 8
 
 def display_data_scenes(nb_bits, p_scene):
@@ -50,21 +44,12 @@ def display_data_scenes(nb_bits, p_scene):
     scenes = [s for s in scenes if min_max_filename not in s]
 
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         if p_scene == folder_scene:
             print(folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             zones_folder = []
 
@@ -77,16 +62,12 @@ def display_data_scenes(nb_bits, p_scene):
                 current_zone = "zone"+index_str
                 zones_folder.append(current_zone)
 
-            zones_images_data = []
             threshold_info = []
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
 
                 zone_path = os.path.join(scene_path, zone_folder)
 
-                current_counter_index = int(start_index_image)
-                end_counter_index = int(end_index_image)
-
                 # get threshold information
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
@@ -101,25 +82,28 @@ def display_data_scenes(nb_bits, p_scene):
             print(mean_threshold, "mean threshold found")
             threshold_image_found = False
 
-            # find appropriate mean threshold picture
-            while(current_counter_index <= end_counter_index and not threshold_image_found):
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_image_path = scene_images[0]
+            end_image_path   = scene_images[-1]
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
 
-                if mean_threshold < int(current_counter_index):
-                    current_counter_index_str = str(current_counter_index)
+            # for each images
+            for img_path in scene_images:
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
-                    while len(start_index_image) > len(current_counter_index_str):
-                        current_counter_index_str = "0" + current_counter_index_str
+                if mean_threshold < int(current_quality_image) and not threshold_image_found:
 
                     threshold_image_found = True
-                    threshold_image_zone = current_counter_index_str
+                    threshold_image_path = img_path
 
-                current_counter_index += step_counter
+                    threshold_image = dt.get_scene_image_quality(img_path)
 
             # all indexes of picture to plot
-            images_indexes = [start_index_image, threshold_image_zone, end_index_image]
-            images_data = []
-
-            print(images_indexes)
+            images_path = [start_image_path, threshold_image_path, end_image_path]
 
             low_bits_svd_values = []
 
@@ -127,16 +111,14 @@ def display_data_scenes(nb_bits, p_scene):
 
                 low_bits_svd_values.append([])
 
-                for index in images_indexes:
-
-                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
+                for img_path in images_path:
 
                     current_img = Image.open(img_path)
 
                     block_used = np.array(current_img)
 
-                    low_bits_block = processing.rgb_to_LAB_L_bits(block_used, (i + 1, i + nb_bits + 1))
-                    low_bits_svd = metrics.get_SVD_s(low_bits_block)
+                    low_bits_block = transform.rgb_to_LAB_L_bits(block_used, (i + 1, i + nb_bits + 1))
+                    low_bits_svd = compression.get_SVD_s(low_bits_block)
                     low_bits_svd = [b / low_bits_svd[0] for b in low_bits_svd]
                     low_bits_svd_values[i].append(low_bits_svd)
 
@@ -146,9 +128,9 @@ def display_data_scenes(nb_bits, p_scene):
 
             for id, data in enumerate(low_bits_svd_values):
                 fig.add_subplot(3, 3, (id + 1))
-                plt.plot(data[0], label='Noisy_' + start_index_image)
-                plt.plot(data[1], label='Threshold_' + threshold_image_zone)
-                plt.plot(data[2], label='Reference_' + end_index_image)
+                plt.plot(data[0], label='Noisy_' + start_quality_image)
+                plt.plot(data[1], label='Threshold_' + threshold_image)
+                plt.plot(data[2], label='Reference_' + end_quality_image)
                 plt.ylabel('Lab SVD ' + str(nb_bits) + ' bits values shifted by ' + str(id), fontsize=14)
                 plt.xlabel('Vector features', fontsize=16)
                 plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)

display/display_scenes_zones.py

@@ -0,0 +1,176 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+import random
+import time
+import json
+
+# image processing imports
+from PIL import Image
+from skimage import color
+import matplotlib.pyplot as plt
+
+from data_attributes import get_svd_data
+
+from ipfml.processing import segmentation, transform, compression
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
+
+# variables and parameters
+zone_folder         = cfg.zone_folder
+min_max_filename    = cfg.min_max_filename_extension
+
+# define all scenes values
+scenes_list         = cfg.scenes_names
+scenes_indices      = cfg.scenes_indices
+norm_choices        = cfg.normalization_choices
+path                = cfg.dataset_path
+zones               = cfg.zones_indices
+seuil_expe_filename = cfg.seuil_expe_filename
+
+features_choices      = cfg.features_choices_labels
+
+
+def display_data_scenes(data_type, p_scene, p_kind):
+    """
+    @brief Method which displays data from scene
+    @param data_type,  feature choice
+    @param scene, scene choice
+    @param mode, normalization choice
+    @return nothing
+    """
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    # go ahead each scenes
+    for folder_scene in scenes:
+
+        if p_scene == folder_scene:
+            print(folder_scene)
+            scene_path = os.path.join(path, folder_scene)
+
+            # construct each zones folder name
+            zones_folder = []
+
+            # get zones list info
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+
+                current_zone = "zone"+index_str
+                zones_folder.append(current_zone)
+
+            zones_images_data = []
+            threshold_info = []
+
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_image_path = scene_images[0]
+            end_image_path   = scene_images[-1]
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
+
+            for id_zone, zone_folder in enumerate(zones_folder):
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                # get threshold information
+                path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+                # open treshold path and get this information
+                with open(path_seuil, "r") as seuil_file:
+                    threshold_learned = int(seuil_file.readline().strip())
+
+                threshold_image_found = False
+
+                for img_path in scene_images:
+                    current_quality_image = dt.get_scene_image_quality(img_path)
+
+                    if threshold_learned < int(current_quality_image) and not threshold_image_found:
+
+                        threshold_image_found = True
+                        threshold_image_path = img_path
+
+                        threshold_image = dt.get_scene_image_postfix(img_path)
+                        threshold_info.append(threshold_image)
+
+                # all indexes of picture to plot
+                images_path = [start_image_path, threshold_image_path, end_image_path]
+                images_data = []
+
+                for img_path in images_path:
+
+                    current_img = Image.open(img_path)
+                    img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
+
+                    # getting expected block id
+                    block = img_blocks[id_zone]
+
+                    data = get_svd_data(data_type, block)
+
+                    ##################
+                    # Data mode part #
+                    ##################
+
+                    # modify data depending mode
+
+                    if p_kind == 'svdn':
+                        data = utils.normalize_arr(data)
+
+                    if p_kind == 'svdne':
+                        path_min_max = os.path.join(path, data_type + min_max_filename)
+
+                        with open(path_min_max, 'r') as f:
+                            min_val = float(f.readline())
+                            max_val = float(f.readline())
+
+                        data = utils.normalize_arr_with_range(data, min_val, max_val)
+
+                    # append of data
+                    images_data.append(data)
+
+                zones_images_data.append(images_data)
+
+            fig=plt.figure(figsize=(8, 8))
+            fig.suptitle(data_type + " values for " + p_scene + " scene (normalization : " + p_kind + ")", fontsize=20)
+
+            for id, data in enumerate(zones_images_data):
+                fig.add_subplot(4, 4, (id + 1))
+                plt.plot(data[0], label='Noisy_' + start_quality_image)
+                plt.plot(data[1], label='Threshold_' + threshold_info[id])
+                plt.plot(data[2], label='Reference_' + end_quality_image)
+                plt.ylabel(data_type + ' SVD, ZONE_' + str(id + 1), fontsize=18)
+                plt.xlabel('Vector features', fontsize=18)
+                plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=18)
+                plt.ylim(0, 0.1)
+            plt.show()
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Display zones curves of feature on scene ")
+
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
+    parser.add_argument('--scene', type=str, help='scene index to use', choices=scenes_indices)
+    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=norm_choices)
+
+    args = parser.parse_args()
+
+    p_feature = args.feature
+    p_kind   = args.kind
+    p_scene  = scenes_list[scenes_indices.index(args.scene)]
+
+    display_data_scenes(p_feature, p_scene, p_kind)
+
+if __name__== "__main__":
+    main()

display_scenes_zones_shifted.py

@@ -1,39 +1,36 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
 import random
 import time
 import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
 from skimage import color
 import matplotlib.pyplot as plt
 
-from modules.utils import config as cfg
+from ipfml.processing import segmentation, transform, compression
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
-config_filename     = cfg.config_filename
+
+# variables and parameters
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 
 # define all scenes values
 scenes_list         = cfg.scenes_names
 scenes_indices      = cfg.scenes_indices
-choices             = cfg.normalization_choices
 path                = cfg.dataset_path
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 
-metric_choices      = cfg.metric_choices_labels
-
 max_nb_bits = 8
 
 def display_data_scenes(p_scene, p_bits, p_shifted):
@@ -50,21 +47,12 @@ def display_data_scenes(p_scene, p_bits, p_shifted):
     scenes = [s for s in scenes if min_max_filename not in s]
 
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         if p_scene == folder_scene:
             print(folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             zones_folder = []
 
@@ -80,55 +68,56 @@ def display_data_scenes(p_scene, p_bits, p_shifted):
             zones_images_data = []
             threshold_info = []
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_image_path = scene_images[0]
+            end_image_path   = scene_images[-1]
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
+
             for id_zone, zone_folder in enumerate(zones_folder):
 
                 zone_path = os.path.join(scene_path, zone_folder)
 
-                current_counter_index = int(start_index_image)
-                end_counter_index = int(end_index_image)
-
                 # get threshold information
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
                 # open treshold path and get this information
                 with open(path_seuil, "r") as seuil_file:
-                    seuil_learned = int(seuil_file.readline().strip())
+                    threshold_learned = int(seuil_file.readline().strip())
 
                 threshold_image_found = False
-                while(current_counter_index <= end_counter_index and not threshold_image_found):
 
-                    if seuil_learned < int(current_counter_index):
-                        current_counter_index_str = str(current_counter_index)
+                # for each images
+                for img_path in scene_images:
+                    current_quality_image = dt.get_scene_image_quality(img_path)
 
-                        while len(start_index_image) > len(current_counter_index_str):
-                            current_counter_index_str = "0" + current_counter_index_str
+                    if threshold_learned < int(current_quality_image) and not threshold_image_found:
 
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
-                        threshold_info.append(threshold_image_zone)
+                        threshold_image_path = img_path
 
-                    current_counter_index += step_counter
+                        threshold_image = dt.get_scene_image_postfix(img_path)
+                        threshold_info.append(threshold_image)
 
                 # all indexes of picture to plot
-                images_indexes = [start_index_image, threshold_image_zone, end_index_image]
+                images_path = [start_image_path, threshold_image_path, end_image_path]
                 images_data = []
 
-                print(images_indexes)
-
-                for index in images_indexes:
-
-                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
+                for img_path in images_path:
 
                     current_img = Image.open(img_path)
-                    img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+                    img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
                     # getting expected block id
                     block = img_blocks[id_zone]
 
                     # get data from mode
                     # Here you can add the way you compute data
-                    low_bits_block = processing.rgb_to_LAB_L_bits(block, (p_shifted + 1, p_shifted + p_bits + 1))
-                    data = metrics.get_SVD_s(low_bits_block)
+                    low_bits_block = transform.rgb_to_LAB_L_bits(block, (p_shifted + 1, p_shifted + p_bits + 1))
+                    data = compression.get_SVD_s(low_bits_block)
 
                     ##################
                     # Data mode part #
@@ -145,9 +134,9 @@ def display_data_scenes(p_scene, p_bits, p_shifted):
 
             for id, data in enumerate(zones_images_data):
                 fig.add_subplot(4, 4, (id + 1))
-                plt.plot(data[0], label='Noisy_' + start_index_image)
+                plt.plot(data[0], label='Noisy_' + start_quality_image)
                 plt.plot(data[1], label='Threshold_' + threshold_info[id])
-                plt.plot(data[2], label='Reference_' + end_index_image)
+                plt.plot(data[2], label='Reference_' + end_quality_image)
                 plt.ylabel('Lab SVD ' + str(p_bits) + ' bits shifted by ' + str(p_shifted) + ', ZONE_' + str(id + 1), fontsize=14)
                 plt.xlabel('Vector features', fontsize=16)
                 plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)

display_simulation_curves.py

@@ -1,13 +1,19 @@
+# main imports
 import numpy as np
 import pandas as pd
+import os, sys, argparse
 
+# image processing imports
 import matplotlib.pyplot as plt
-import os, sys, argparse
+from data_attributes import get_svd_data
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-from modules.utils.data import get_svd_data
+import custom_config as cfg
 
-from modules.utils import config as cfg
 
+# variables and parameters
 learned_zones_folder = cfg.learned_zones_folder
 models_name          = cfg.models_names_list
 label_freq           = 6
@@ -25,7 +31,7 @@ def display_curves(folder_path, model_name):
             data_filename = model_name
             learned_zones_folder_path = os.path.join(learned_zones_folder, data_filename)
 
-    data_files = [x for x in os.listdir(folder_path) if '.png' not in x]
+    data_files = [x for x in os.listdir(folder_path) if cfg.scene_image_extension not in x]
 
     scene_names = [f.split('_')[3] for f in data_files]
 

display_svd_area_data_scene.py

@@ -1,32 +1,25 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
-
 import numpy as np
-import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
-
 from skimage import color
-
 import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+from ipfml.processing import segmentation, transform, compression
+from ipfml import utils
+import ipfml.iqa.fr as fr_iqa
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 
@@ -38,7 +31,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 max_nb_bits = 8
 
@@ -46,7 +39,6 @@ integral_area_choices = ['trapz', 'simps']
 
 def get_area_under_curve(p_area, p_data):
 
-    noise_method = None
     function_name = 'integral_area_' + p_area
 
     try:
@@ -74,8 +66,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     max_value_svd = 0
     min_value_svd = sys.maxsize
 
-    image_indices = []
-
     scenes = os.listdir(path)
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
@@ -83,23 +73,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             zones_folder = []
 
@@ -117,6 +96,10 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
             threshold_learned_zones = []
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+            number_scene_image = len(scene_images)
+
             for id, zone_folder in enumerate(zones_folder):
 
                 # get threshold information
@@ -128,25 +111,17 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned_zones.append(threshold_learned)
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_image_found = False
 
-            file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
             svd_data = []
 
-            while(current_counter_index <= end_counter_index):
-
-                current_counter_index_str = str(current_counter_index)
-
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+            # for each images
+            for id_img, img_path in enumerate(scene_images):
+                
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
-                image_path = file_path.format(str(current_counter_index_str))
-                img = Image.open(image_path)
+                img = Image.open(img_path)
 
                 svd_values = get_svd_data(p_metric, img)
 
@@ -164,25 +139,24 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     max_value_svd = max_value
 
                 # keep in memory used data
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+                        images_indices.append(current_quality_image)
                         svd_data.append(svd_values)
 
-                    if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                    if threshold_mean < current_quality_image and not threshold_image_found:
 
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
+                        image_name_postfix = dt.get_scene_image_postfix(img_path)
+                        threshold_image_zone = image_name_postfix
 
-                current_counter_index += step_counter
-                print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+                print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
                 sys.stdout.write("\033[F")
 
 
             # all indices of picture to plot
             print(images_indices)
 
-            previous_data = []
             area_data = []
 
             for id, data in enumerate(svd_data):
@@ -249,7 +223,7 @@ def main():
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='Feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -261,14 +235,14 @@ def main():
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_mode     = args.mode
     p_step     = args.step
     p_norm     = args.norm
     p_area     = args.area
     p_ylim     = list(map(int, args.ylim.split(',')))
 
-    display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_area, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_area, p_ylim)
 
 if __name__== "__main__":
     main()

display_svd_area_scenes.py

@@ -1,32 +1,23 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
-
 import numpy as np
-import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
+import matplotlib.pyplot as plt
 
-from skimage import color
+from data_attributes import get_svd_data
 
-import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
 
-from modules.utils import config as cfg
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 
@@ -38,7 +29,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 max_nb_bits = 8
 
@@ -46,7 +37,6 @@ integral_area_choices = ['trapz', 'simps']
 
 def get_area_under_curve(p_area, p_data):
 
-    noise_method = None
     function_name = 'integral_area_' + p_area
 
     try:
@@ -60,10 +50,9 @@ def get_area_under_curve(p_area, p_data):
 def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_area, p_ylim):
     """
     @brief Method which gives information about svd curves from zone of picture
-    @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
-    @param p_interval, interval [begin, end] of samples or minutes from render generation engine
-    @param p_metric, metric computed to show
+    @param p_indices, indices to display
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_norm, normalization or not of selected svd data
     @param p_area, area method name to compute area under curve
@@ -80,30 +69,19 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # Store all informations about scenes
     scenes_area_data = []
     scenes_images_indices = []
     scenes_threshold_mean = []
 
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         max_value_svd = 0
         min_value_svd = sys.maxsize
 
         scene_path = os.path.join(path, folder_scene)
 
-        config_file_path = os.path.join(scene_path, config_filename)
-
-        with open(config_file_path, "r") as config_file:
-            last_image_name = config_file.readline().strip()
-            prefix_image_name = config_file.readline().strip()
-            start_index_image = config_file.readline().strip()
-            end_index_image = config_file.readline().strip()
-            step_counter = int(config_file.readline().strip())
-
         # construct each zones folder name
         zones_folder = []
 
@@ -121,6 +99,10 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
         images_indices = []
         threshold_learned_zones = []
 
+        # get all images of folder
+        scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+        number_scene_image = len(scene_images)
+
         for id, zone_folder in enumerate(zones_folder):
 
             # get threshold information
@@ -132,26 +114,18 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
                 threshold_learned = int(seuil_file.readline().strip())
                 threshold_learned_zones.append(threshold_learned)
 
-        current_counter_index = int(start_index_image)
-        end_counter_index = int(end_index_image)
-
         threshold_mean = np.mean(np.asarray(threshold_learned_zones))
         threshold_image_found = False
         scenes_threshold_mean.append(int(threshold_mean / p_step))
 
-        file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
         svd_data = []
 
-        while(current_counter_index <= end_counter_index):
-
-            current_counter_index_str = str(current_counter_index)
+        # for each images
+        for id_img, img_path in enumerate(scene_images):
+            
+            current_quality_image = dt.get_scene_image_quality(img_path)
 
-            while len(start_index_image) > len(current_counter_index_str):
-                current_counter_index_str = "0" + current_counter_index_str
-
-            image_path = file_path.format(str(current_counter_index_str))
-            img = Image.open(image_path)
+            img = Image.open(img_path)
 
             svd_values = get_svd_data(p_metric, img)
 
@@ -169,25 +143,22 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
                 max_value_svd = max_value
 
             # keep in memory used data
-            if current_counter_index % p_step == 0:
-                if current_counter_index >= begin_index and current_counter_index <= end_index:
-                    images_indices.append(current_counter_index_str)
+            if current_quality_image % p_step == 0:
+                if current_quality_image >= begin_index and current_quality_image <= end_index:
+                    images_indices.append(dt.get_scene_image_postfix(img_path))
                     svd_data.append(svd_values)
 
-                if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                if threshold_mean < current_quality_image and not threshold_image_found:
 
                     threshold_image_found = True
-                    threshold_image_zone = current_counter_index_str
 
-            current_counter_index += step_counter
-            print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+            print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
             sys.stdout.write("\033[F")
 
 
             # all indices of picture to plot
         print("Scene %s : %s" % (folder_scene, images_indices))
 
-
         scenes_images_indices.append(image_indices)
 
         area_data = []
@@ -225,14 +196,12 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
         threshold_id = 0
         scene_name = scenes[id]
         image_indices = scenes_images_indices[id]
-        threshold_image_zone = scenes_threshold_mean[id]
 
         p_label = scene_name + '_' + str(images_indices[id])
 
         threshold_id = scenes_threshold_mean[id]
 
         print(p_label)
-        start_ylim, end_ylim = p_ylim
 
         plt.plot(area_data, label=p_label)
         #ax2.set_xticks(range(len(images_indices)))
@@ -242,8 +211,8 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
             plt.plot([threshold_id, threshold_id], [np.min(area_data), np.max(area_data)], 'k-', lw=2, color='red')
 
 
-    #start_ylim, end_ylim = p_ylim
-    #plt.ylim(start_ylim, end_ylim)
+    start_ylim, end_ylim = p_ylim
+    plt.ylim(start_ylim, end_ylim)
 
     plt.show()
 
@@ -251,10 +220,10 @@ def main():
 
     parser = argparse.ArgumentParser(description="Display area under curve on scene")
 
-    parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
+    #parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='Metric data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -263,17 +232,17 @@ def main():
 
     args = parser.parse_args()
 
-    p_scene    = scenes_list[scenes_indices.index(args.scene)]
+    #p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_mode     = args.mode
     p_step     = args.step
     p_norm     = args.norm
     p_area     = args.area
     p_ylim     = list(map(int, args.ylim.split(',')))
 
-    display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_area, p_ylim)
+    display_svd_values(p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_area, p_ylim)
 
 if __name__== "__main__":
     main()

display_svd_data_error_scene.py

@@ -1,32 +1,24 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
-
 import numpy as np
-import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
-
 from skimage import color
-
 import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 
@@ -38,7 +30,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 max_nb_bits         = 8
 display_error       = False
@@ -48,7 +40,6 @@ error_data_choices  = ['mae', 'mse', 'ssim', 'psnr']
 
 def get_error_distance(p_error, y_true, y_test):
 
-    noise_method = None
     function_name = p_error
 
     try:
@@ -59,16 +50,16 @@ def get_error_distance(p_error, y_true, y_test):
     return error_method(y_true, y_test)
 
 
-def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_error, p_ylim):
+def display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_error, p_ylim):
     """
     @brief Method which gives information about svd curves from zone of picture
     @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
-    @param p_metric, metric computed to show
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_norm, normalization or not of selected svd data
-    @param p_error, error metric used to display
+    @param p_error, error feature used to display
     @param p_ylim, ylim choice to better display of data
     @return nothing
     """
@@ -76,8 +67,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     max_value_svd = 0
     min_value_svd = sys.maxsize
 
-    image_indices = []
-
     scenes = os.listdir(path)
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
@@ -85,23 +74,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             zones_folder = []
 
@@ -115,10 +93,14 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                 zones_folder.append(current_zone)
 
             images_data = []
-            images_indices = []
+            images_path = []
 
             threshold_learned_zones = []
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+            number_scene_image = len(scene_images)
+
             for id, zone_folder in enumerate(zones_folder):
 
                 # get threshold information
@@ -131,27 +113,19 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned_zones.append(threshold_learned)
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_image_found = False
 
-            file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
             svd_data = []
+           
+            # for each images
+            for id_img, img_path in enumerate(scene_images):
+                
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
-            while(current_counter_index <= end_counter_index):
-
-                current_counter_index_str = str(current_counter_index)
-
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+                img = Image.open(img_path)
 
-                image_path = file_path.format(str(current_counter_index_str))
-                img = Image.open(image_path)
-
-                svd_values = get_svd_data(p_metric, img)
+                svd_values = get_svd_data(p_feature, img)
 
                 if p_norm:
                     svd_values = svd_values[begin_data:end_data]
@@ -167,23 +141,19 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     max_value_svd = max_value
 
                 # keep in memory used data
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+                        images_path.append(img_path)
                         svd_data.append(svd_values)
 
-                    if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                    if threshold_mean < current_quality_image and not threshold_image_found:
 
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
+                        threshold_image_zone = dt.get_scene_image_postfix(img_path)
 
-                current_counter_index += step_counter
-                print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+                print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
                 sys.stdout.write("\033[F")
 
-            # all indices of picture to plot
-            print(images_indices)
-
             previous_data = []
             error_data = [0.]
 
@@ -204,8 +174,7 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
                 # use of whole image data for computation of ssim or psnr
                 if p_error == 'ssim' or p_error == 'psnr':
-                    image_path = file_path.format(str(images_indices[id]))
-                    current_data = np.asarray(Image.open(image_path))
+                    current_data = np.asarray(Image.open(images_path[id]))
 
                 if len(previous_data) > 0:
 
@@ -224,18 +193,21 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             ax2 = plt.subplot2grid(gridsize, (2, 0), colspan=2)
 
 
-            ax1.set_title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_metric + ' metric, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
+            ax1.set_title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_feature + ' feature, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
             ax1.set_ylabel('Image samples or time (minutes) generation', fontsize=14)
             ax1.set_xlabel('Vector features', fontsize=16)
 
             for id, data in enumerate(images_data):
+                
+                current_quality_image = dt.get_scene_image_quality(images_path[id])
+                current_quality_postfix = dt.get_scene_image_postfix(images_path[id])
 
                 if display_error:
-                    p_label = p_scene + '_' + str(images_indices[id]) + " | " + p_error + ": " + str(error_data[id])
+                    p_label = p_scene + '_' + current_quality_postfix + " | " + p_error + ": " + str(error_data[id])
                 else:
-                    p_label = p_scene + '_' + str(images_indices[id])
+                    p_label = p_scene + '_' + current_quality_postfix
 
-                if images_indices[id] == threshold_image_zone:
+                if current_quality_image == threshold_image_zone:
                     ax1.plot(data, label=p_label + " (threshold mean)", lw=4, color='red')
                 else:
                     ax1.plot(data, label=p_label)
@@ -248,11 +220,11 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             ax2.set_title(p_error + " information for whole step images")
             ax2.set_ylabel(p_error + ' error')
             ax2.set_xlabel('Number of samples per pixels or times')
-            ax2.set_xticks(range(len(images_indices)))
-            ax2.set_xticklabels(list(map(int, images_indices)))
+            ax2.set_xticks(range(len(current_quality_image)))
+            ax2.set_xticklabels(list(map(dt.get_scene_image_quality, current_quality_image)))
             ax2.plot(error_data)
 
-            plot_name = p_scene + '_' + p_metric + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
+            plot_name = p_scene + '_' + p_feature + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
             plt.savefig(plot_name)
 
 def main():
@@ -262,7 +234,7 @@ def main():
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -274,14 +246,14 @@ def main():
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature   = args.feature
     p_mode     = args.mode
     p_step     = args.step
     p_norm     = args.norm
     p_error    = args.error
     p_ylim     = list(map(int, args.ylim.split(',')))
 
-    display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_error, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_error, p_ylim)
 
 if __name__== "__main__":
     main()

display_svd_data_scene.py

@@ -1,34 +1,23 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
-
 import numpy as np
-import random
-import time
-import json
-import math
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
+import matplotlib.pyplot as plt
 
-from skimage import color
+from data_attributes import get_svd_data
 
-import matplotlib as mpl
-import matplotlib.pyplot as plt
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
 
-from modules.utils.data import get_svd_data
-from modules.utils import config as cfg
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 
@@ -40,19 +29,19 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 max_nb_bits         = 8
 display_error       = False
 
 
-def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_ylim):
+def display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_ylim):
     """
     @brief Method which gives information about svd curves from zone of picture
     @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
-    @param p_metric, metric computed to show
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_norm, normalization or not of selected svd data
     @param p_ylim, ylim choice to better display of data
@@ -62,8 +51,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     max_value_svd = 0
     min_value_svd = sys.maxsize
 
-    image_indices = []
-
     scenes = os.listdir(path)
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
@@ -71,23 +58,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             zones_folder = []
 
@@ -104,11 +80,14 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             images_indices = []
 
             threshold_learned_zones = []
-
+    
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+            number_scene_image = len(scene_images)
+            
             for id, zone_folder in enumerate(zones_folder):
 
                 # get threshold information
-
                 zone_path = os.path.join(scene_path, zone_folder)
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
@@ -117,27 +96,20 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned_zones.append(threshold_learned)
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_image_found = False
 
-            file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
             svd_data = []
 
-            while(current_counter_index <= end_counter_index):
 
-                current_counter_index_str = str(current_counter_index)
+            # for each images
+            for id_img, img_path in enumerate(scene_images):
+                
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+                img = Image.open(img_path)
 
-                image_path = file_path.format(str(current_counter_index_str))
-                img = Image.open(image_path)
-
-                svd_values = get_svd_data(p_metric, img)
+                svd_values = get_svd_data(p_feature, img)
 
                 if p_norm:
                     svd_values = svd_values[begin_data:end_data]
@@ -155,18 +127,18 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     max_value_svd = max_value
 
                 # keep in memory used data
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+
+                        images_indices.append(dt.get_scene_image_postfix(img_path))
                         svd_data.append(svd_values)
 
-                    if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                    if threshold_mean < current_quality_image and not threshold_image_found:
 
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
+                        threshold_image_zone = current_quality_image
 
-                current_counter_index += step_counter
-                print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+                print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
                 sys.stdout.write("\033[F")
 
 
@@ -190,7 +162,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
 
             # display all data using matplotlib (configure plt)
-            #fig = plt.figure(figsize=(30, 22))
             fig, ax = plt.subplots(figsize=(30, 22))
             ax.set_facecolor('#F9F9F9')
             #fig.patch.set_facecolor('#F9F9F9')
@@ -199,7 +170,7 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             #plt.rc('xtick', labelsize=22)
             #plt.rc('ytick', labelsize=22)
 
-            #plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_metric + ' metric, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=24)
+            #plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_feature + ' feature, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=24)
             ax.set_ylabel('Component values', fontsize=30)
             ax.set_xlabel('Vector features', fontsize=30)
 
@@ -214,10 +185,10 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
             plt.legend(bbox_to_anchor=(0.65, 0.98), loc=2, borderaxespad=0.2, fontsize=24)
 
-            start_ylim, end_ylim = p_ylim
+            #start_ylim, end_ylim = p_ylim
             #ax.set_ylim(start_ylim, end_ylim)
 
-            plot_name = p_scene + '_' + p_metric + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
+            plot_name = p_scene + '_' + p_feature + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
             plt.savefig(plot_name, facecolor=ax.get_facecolor())
 
 def main():
@@ -227,7 +198,7 @@ def main():
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -238,13 +209,13 @@ def main():
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_mode     = args.mode
     p_step     = args.step
     p_norm     = args.norm
     p_ylim     = list(map(int, args.ylim.split(',')))
 
-    display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_ylim)
 
 if __name__== "__main__":
     main()

display_svd_zone_scene.py

@@ -1,30 +1,24 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
-
 import numpy as np
-import random
-import time
-import json
 
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
-from skimage import color
-
 import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+from ipfml.processing import segmentation
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 
@@ -36,7 +30,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 generic_output_file_svd = '_random.csv'
 
@@ -44,7 +38,7 @@ max_nb_bits = 8
 min_value_interval = sys.maxsize
 max_value_interval = 0
 
-def get_min_max_value_interval(_scene, _interval, _metric):
+def get_min_max_value_interval(_scene, _interval, _feature):
 
     global min_value_interval, max_value_interval
 
@@ -53,7 +47,7 @@ def get_min_max_value_interval(_scene, _interval, _metric):
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
 
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         # only take care of current scene
         if folder_scene == _scene:
@@ -68,9 +62,9 @@ def get_min_max_value_interval(_scene, _interval, _metric):
                     index_str = "0" + index_str
                 zones_folder.append("zone"+index_str)
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
                 zone_path = os.path.join(scene_path, zone_folder)
-                data_filename = _metric + "_svd" + generic_output_file_svd
+                data_filename = _feature + "_svd" + generic_output_file_svd
                 data_file_path = os.path.join(zone_path, data_filename)
 
                 # getting number of line and read randomly lines
@@ -83,11 +77,11 @@ def get_min_max_value_interval(_scene, _interval, _metric):
                     begin, end = _interval
 
                     line_data = line.split(';')
-                    metrics = line_data[begin+1:end+1]
-                    metrics = [float(m) for m in metrics]
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
 
-                    min_value = min(metrics)
-                    max_value = max(metrics)
+                    min_value = min(features)
+                    max_value = max(features)
 
                     if min_value < min_value_interval:
                         min_value_interval = min_value
@@ -96,14 +90,14 @@ def get_min_max_value_interval(_scene, _interval, _metric):
                         max_value_interval = max_value
 
 
-def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode, p_step, p_norm, p_ylim):
+def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_feature, p_mode, p_step, p_norm, p_ylim):
     """
     @brief Method which gives information about svd curves from zone of picture
     @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
     @param p_zone, zone's identifier of picture
-    @param p_metric, metric computed to show
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_step, step of images indices
     @param p_norm, normalization or not of selected svd data
@@ -118,23 +112,14 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
+    data_min_max_filename = os.path.join(path, p_feature + min_max_filename)
 
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             zones_folder = []
 
@@ -148,15 +133,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
                 zones_folder.append(current_zone)
 
             zones_images_data = []
-            images_indices = []
+            images_path = []
 
             zone_folder = zones_folder[p_zone]
 
             zone_path = os.path.join(scene_path, zone_folder)
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             # get threshold information
             path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
@@ -166,40 +148,35 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
 
             threshold_image_found = False
 
-            while(current_counter_index <= end_counter_index):
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
 
-                current_counter_index_str = str(current_counter_index)
+            # for each images
+            for img_path in scene_images:
+                    
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+                        images_path.append(dt.get_scene_image_postfix(img_path))
 
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
-
-                    if seuil_learned < int(current_counter_index) and not threshold_image_found:
+                    if seuil_learned < current_quality_image and not threshold_image_found:
 
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
-
-                current_counter_index += step_counter
-
-            # all indices of picture to plot
-            print(images_indices)
+                        threshold_image_zone = dt.get_scene_image_postfix(img_path)
 
-            for index in images_indices:
 
-                img_path = os.path.join(scene_path, prefix_image_name + str(index) + ".png")
+            for img_path in images_path:
 
                 current_img = Image.open(img_path)
-                img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+                img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
                 # getting expected block id
                 block = img_blocks[p_zone]
 
                 # get data from mode
                 # Here you can add the way you compute data
-                data = get_svd_data(p_metric, block)
+                data = get_svd_data(p_feature, block)
 
                 # TODO : improve part of this code to get correct min / max values
                 if p_norm:
@@ -230,15 +207,15 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
                 else:
                     zones_images_data.append(data)
 
-            plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + ']' + p_metric + ' metric, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
+            plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + ']' + p_feature + ' feature, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
             plt.ylabel('Image samples or time (minutes) generation', fontsize=14)
             plt.xlabel('Vector features', fontsize=16)
 
             for id, data in enumerate(zones_images_data):
 
-                p_label = p_scene + "_" + images_indices[id]
+                p_label = p_scene + "_" + images_path[id]
 
-                if images_indices[id] == threshold_image_zone:
+                if images_path[id] == threshold_image_zone:
                     plt.plot(data, label=p_label, lw=4, color='red')
                 else:
                     plt.plot(data, label=p_label)
@@ -258,7 +235,7 @@ def main():
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
     parser.add_argument('--zone', type=int, help='Zone to display', choices=list(range(0, 16)))
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -270,13 +247,13 @@ def main():
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
     p_zone     = args.zone
-    p_metric   = args.metric
+    p_feature   = args.feature
     p_mode     = args.mode
     p_step     = args.step
     p_norm     = args.norm
     p_ylim     = list(map(int, args.ylim.split(',')))
 
-    display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode, p_step, p_norm, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_zone, p_feature, p_mode, p_step, p_norm, p_ylim)
 
 if __name__== "__main__":
     main()

display_scenes_zones.py

@@ -1,249 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
-import sys, os, argparse
-import numpy as np
-import random
-import time
-import json
-
-from PIL import Image
-from ipfml import processing, metrics, utils
-from skimage import color
-import matplotlib.pyplot as plt
-
-from modules.utils import config as cfg
-
-config_filename     = cfg.config_filename
-zone_folder         = cfg.zone_folder
-min_max_filename    = cfg.min_max_filename_extension
-
-# define all scenes values
-scenes_list         = cfg.scenes_names
-scenes_indices      = cfg.scenes_indices
-norm_choices        = cfg.normalization_choices
-path                = cfg.dataset_path
-zones               = cfg.zones_indices
-seuil_expe_filename = cfg.seuil_expe_filename
-
-metric_choices      = cfg.metric_choices_labels
-
-
-def display_data_scenes(data_type, p_scene, p_kind):
-    """
-    @brief Method which displays data from scene
-    @param data_type,  metric choice
-    @param scene, scene choice
-    @param mode, normalization choice
-    @return nothing
-    """
-
-    scenes = os.listdir(path)
-    # remove min max file from scenes folder
-    scenes = [s for s in scenes if min_max_filename not in s]
-
-    # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
-
-        if p_scene == folder_scene:
-            print(folder_scene)
-            scene_path = os.path.join(path, folder_scene)
-
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
-            # construct each zones folder name
-            zones_folder = []
-
-            # get zones list info
-            for index in zones:
-                index_str = str(index)
-                if len(index_str) < 2:
-                    index_str = "0" + index_str
-
-                current_zone = "zone"+index_str
-                zones_folder.append(current_zone)
-
-            zones_images_data = []
-            threshold_info = []
-
-            for id_zone, zone_folder in enumerate(zones_folder):
-
-                zone_path = os.path.join(scene_path, zone_folder)
-
-                current_counter_index = int(start_index_image)
-                end_counter_index = int(end_index_image)
-
-                # get threshold information
-                path_seuil = os.path.join(zone_path, seuil_expe_filename)
-
-                # open treshold path and get this information
-                with open(path_seuil, "r") as seuil_file:
-                    seuil_learned = int(seuil_file.readline().strip())
-
-                threshold_image_found = False
-                while(current_counter_index <= end_counter_index and not threshold_image_found):
-
-                    if seuil_learned < int(current_counter_index):
-                        current_counter_index_str = str(current_counter_index)
-
-                        while len(start_index_image) > len(current_counter_index_str):
-                            current_counter_index_str = "0" + current_counter_index_str
-
-                        threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
-                        threshold_info.append(threshold_image_zone)
-
-                    current_counter_index += step_counter
-
-                # all indexes of picture to plot
-                images_indexes = [start_index_image, threshold_image_zone, end_index_image]
-                images_data = []
-
-                print(images_indexes)
-
-                for index in images_indexes:
-
-                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
-
-                    current_img = Image.open(img_path)
-                    img_blocks = processing.divide_in_blocks(current_img, (200, 200))
-
-                    # getting expected block id
-                    block = img_blocks[id_zone]
-
-                    # get data from mode
-                    # Here you can add the way you compute data
-                    if data_type == 'lab':
-
-                        block_file_path = '/tmp/lab_img.png'
-                        block.save(block_file_path)
-                        data = processing.get_LAB_L_SVD_s(Image.open(block_file_path))
-
-                    if data_type == 'mscn_revisited':
-
-                        img_mscn_revisited = processing.rgb_to_mscn(block)
-
-                        # save tmp as img
-                        img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
-                        mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
-                        img_output.save(mscn_revisited_file_path)
-                        img_block = Image.open(mscn_revisited_file_path)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(img_block)
-
-                    '''if data_type == 'mscn':
-
-                        img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
-                        img_mscn = processing.calculate_mscn_coefficients(img_gray, 7)
-                        img_mscn_norm = utils.normalize_2D_arr(img_mscn)
-
-                        img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
-
-                        data = metrics.get_SVD_s(img_mscn_gray)'''
-
-                    if data_type == 'low_bits_6':
-
-                        low_bits_6 = processing.rgb_to_LAB_L_low_bits(block, 63)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_6)
-
-
-                    if data_type == 'low_bits_5':
-
-                        low_bits_5 = processing.rgb_to_LAB_L_low_bits(block, 31)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_5)
-
-
-                    if data_type == 'low_bits_4':
-
-                        low_bits_4 = processing.rgb_to_LAB_L_low_bits(block)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_4)
-
-                    if data_type == 'low_bits_3':
-
-                        low_bits_3 = processing.rgb_to_LAB_L_low_bits(block, 7)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_3)
-
-                    if data_type == 'low_bits_2':
-
-                        low_bits_2 = processing.rgb_to_LAB_L_low_bits(block, 3)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_2)
-
-                    ##################
-                    # Data mode part #
-                    ##################
-
-                    # modify data depending mode
-
-                    if p_kind == 'svdn':
-                        data = utils.normalize_arr(data)
-
-                    if p_kind == 'svdne':
-                        path_min_max = os.path.join(path, data_type + min_max_filename)
-
-                        with open(path_min_max, 'r') as f:
-                            min_val = float(f.readline())
-                            max_val = float(f.readline())
-
-                        data = utils.normalize_arr_with_range(data, min_val, max_val)
-
-                    # append of data
-                    images_data.append(data)
-
-                zones_images_data.append(images_data)
-
-            fig=plt.figure(figsize=(8, 8))
-            fig.suptitle(data_type + " values for " + p_scene + " scene (normalization : " + p_kind + ")", fontsize=20)
-
-            for id, data in enumerate(zones_images_data):
-                fig.add_subplot(4, 4, (id + 1))
-                plt.plot(data[0], label='Noisy_' + start_index_image)
-                plt.plot(data[1], label='Threshold_' + threshold_info[id])
-                plt.plot(data[2], label='Reference_' + end_index_image)
-                plt.ylabel(data_type + ' SVD, ZONE_' + str(id + 1), fontsize=18)
-                plt.xlabel('Vector features', fontsize=18)
-                plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=18)
-                plt.ylim(0, 0.1)
-            plt.show()
-
-def main():
-
-    parser = argparse.ArgumentParser(description="Display zones curves of metric on scene ")
-
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
-    parser.add_argument('--scene', type=str, help='scene index to use', choices=scenes_indices)
-    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=norm_choices)
-
-    args = parser.parse_args()
-
-    p_metric = args.metric
-    p_kind   = args.kind
-    p_scene  = scenes_list[scenes_indices.index(args.scene)]
-
-    display_data_scenes(p_metric, p_scene, p_kind)
-
-if __name__== "__main__":
-    main()

generate_all_data.py

@@ -1,27 +1,27 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import numpy as np
 import random
 import time
 import json
 
-from modules.utils.data import get_svd_data
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
 from skimage import color
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+from ipfml.processing import transform, segmentation
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
 
 # getting configuration information
-config_filename         = cfg.config_filename
 zone_folder             = cfg.zone_folder
 min_max_filename        = cfg.min_max_filename_extension
 
@@ -33,7 +33,7 @@ path                    = cfg.dataset_path
 zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 
-metric_choices          = cfg.metric_choices_labels
+features_choices        = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 
 generic_output_file_svd = '_random.csv'
@@ -41,7 +41,7 @@ generic_output_file_svd = '_random.csv'
 def generate_data_svd(data_type, mode):
     """
     @brief Method which generates all .csv files from scenes
-    @param data_type,  metric choice
+    @param data_type,  feature choice
     @param mode, normalization choice
     @return nothing
     """
@@ -57,20 +57,11 @@ def generate_data_svd(data_type, mode):
     data_min_max_filename = os.path.join(path, data_type + min_max_filename)
 
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
         print(folder_scene)
         scene_path = os.path.join(path, folder_scene)
 
-        config_file_path = os.path.join(scene_path, config_filename)
-
-        with open(config_file_path, "r") as config_file:
-            last_image_name = config_file.readline().strip()
-            prefix_image_name = config_file.readline().strip()
-            start_index_image = config_file.readline().strip()
-            end_index_image = config_file.readline().strip()
-            step_counter = int(config_file.readline().strip())
-
         # getting output filename
         output_svd_filename = data_type + "_" + mode + generic_output_file_svd
 
@@ -93,27 +84,21 @@ def generate_data_svd(data_type, mode):
             # add writer into list
             svd_output_files.append(open(svd_file_path, 'w'))
 
-
-        current_counter_index = int(start_index_image)
-        end_counter_index = int(end_index_image)
-
-
-        while(current_counter_index <= end_counter_index):
-
-            current_counter_index_str = str(current_counter_index)
-
-            while len(start_index_image) > len(current_counter_index_str):
-                current_counter_index_str = "0" + current_counter_index_str
-
-            img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+        # get all images of folder
+        scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+        number_scene_image = len(scene_images)
+            
+        for id_img, img_path in enumerate(scene_images):
+            
+            current_image_postfix = dt.get_scene_image_postfix(img_path)
 
             current_img = Image.open(img_path)
-            img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+            img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
             for id_block, block in enumerate(img_blocks):
 
                 ###########################
-                # Metric computation part #
+                # feature computation part #
                 ###########################
 
                 data = get_svd_data(data_type, block)
@@ -151,19 +136,16 @@ def generate_data_svd(data_type, mode):
                 current_file = svd_output_files[id_block]
 
                 # add of index
-                current_file.write(current_counter_index_str + ';')
+                current_file.write(current_image_postfix + ';')
 
                 for val in data:
                     current_file.write(str(val) + ";")
 
                 current_file.write('\n')
 
-            start_index_image_int = int(start_index_image)
-            print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((current_counter_index - start_index_image_int) / (end_counter_index - start_index_image_int)* 100.) + "%")
+            print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((id_img + 1) / number_scene_image * 100.) + "%")
             sys.stdout.write("\033[F")
 
-            current_counter_index += step_counter
-
         for f in svd_output_files:
             f.close()
 
@@ -180,26 +162,26 @@ def generate_data_svd(data_type, mode):
 
 def main():
 
-    parser = argparse.ArgumentParser(description="Compute and prepare data of metric of all scenes (keep in memory min and max value found)")
+    parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes (keep in memory min and max value found)")
 
-    parser.add_argument('--metric', type=str, 
-                                    help="metric choice in order to compute data (use 'all' if all metrics are needed)", 
-                                    choices=metric_choices)
+    parser.add_argument('--feature', type=str, 
+                                    help="feature choice in order to compute data (use 'all' if all features are needed)", 
+                                    choices=features_choices)
 
     args = parser.parse_args()
 
-    p_metric = args.metric
+    p_feature = args.feature
 
-    # generate all or specific metric data
-    if p_metric == 'all':
-        for m in metric_choices:
+    # generate all or specific feature data
+    if p_feature == 'all':
+        for m in features_choices:
             generate_data_svd(m, 'svd')
             generate_data_svd(m, 'svdn')
             generate_data_svd(m, 'svdne')
     else:
-        generate_data_svd(p_metric, 'svd')
-        generate_data_svd(p_metric, 'svdn')
-        generate_data_svd(p_metric, 'svdne')
+        generate_data_svd(p_feature, 'svd')
+        generate_data_svd(p_feature, 'svdn')
+        generate_data_svd(p_feature, 'svdne')
 
 if __name__== "__main__":
     main()

modules

@@ -0,0 +1 @@
+Subproject commit 139aa3c2312e9449b32d1d6fa506d741e7790c98

modules/models.py

@@ -1,75 +0,0 @@
-from sklearn.model_selection import GridSearchCV
-from sklearn.linear_model import LogisticRegression
-from sklearn.ensemble import RandomForestClassifier, VotingClassifier
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.ensemble import GradientBoostingClassifier
-import sklearn.svm as svm
-
-
-def _get_best_model(X_train, y_train):
-
-    Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
-    gammas = [0.001, 0.01, 0.1, 1, 5, 10, 100]
-    param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
-
-    svc = svm.SVC(probability=True)
-    clf = GridSearchCV(svc, param_grid, cv=10, scoring='accuracy', verbose=10)
-
-    clf.fit(X_train, y_train)
-
-    model = clf.best_estimator_
-
-    return model
-
-def svm_model(X_train, y_train):
-
-    return _get_best_model(X_train, y_train)
-
-
-def ensemble_model(X_train, y_train):
-
-    svm_model = _get_best_model(X_train, y_train)
-
-    lr_model = LogisticRegression(solver='liblinear', multi_class='ovr', random_state=1)
-    rf_model = RandomForestClassifier(n_estimators=100, random_state=1)
-
-    ensemble_model = VotingClassifier(estimators=[
-       ('svm', svm_model), ('lr', lr_model), ('rf', rf_model)], voting='soft', weights=[1,1,1])
-
-    ensemble_model.fit(X_train, y_train)
-
-    return ensemble_model
-
-
-def ensemble_model_v2(X_train, y_train):
-
-    svm_model = _get_best_model(X_train, y_train)
-    knc_model = KNeighborsClassifier(n_neighbors=2)
-    gbc_model = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)
-    lr_model = LogisticRegression(solver='liblinear', multi_class='ovr', random_state=1)
-    rf_model = RandomForestClassifier(n_estimators=100, random_state=1)
-
-    ensemble_model = VotingClassifier(estimators=[
-       ('lr', lr_model),
-       ('knc', knc_model),
-       ('gbc', gbc_model),
-       ('svm', svm_model),
-       ('rf', rf_model)],
-       voting='soft', weights=[1, 1, 1, 1, 1])
-
-    ensemble_model.fit(X_train, y_train)
-
-    return ensemble_model
-
-def get_trained_model(choice, X_train, y_train):
-
-    if choice == 'svm_model':
-        return svm_model(X_train, y_train)
-
-    if choice == 'ensemble_model':
-        return ensemble_model(X_train, y_train)
-
-    if choice == 'ensemble_model_v2':
-        return ensemble_model_v2(X_train, y_train)
-
-

predict_seuil_expe_maxwell_curve.py

@@ -124,9 +124,8 @@ def main():
                         tmp_file_path = tmp_filename.replace('__model__',  p_model_file.split('/')[-1].replace('.joblib', '_'))
                         block.save(tmp_file_path)
 
-                        python_cmd = """python predict_noisy_image_svd.py --image {0} --interval '{1}' --model {2} --mode {3} --metric {4}""".format(tmp_file_path, p_interval, p_model_file, p_mode, p_metric)
-
-                        print(python_cmd)
+                        python_cmd_line = "python predict_noisy_image_svd.py --image {0} --interval '{1}' --model {2} --mode {3} --metric {4}"
+                        python_cmd = python_cmd_line.format(tmp_file_path, p_interval, p_model_file, p_mode, p_metric) 
 
                         # specify use of custom file for min max normalization
                         if p_custom:

deep_network_keras_svd.py

@@ -1,3 +1,11 @@
+# main imports
+import sys, os
+import argparse
+import json
+import numpy as np
+import pandas as pd
+
+# models imports
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
 from keras.layers import Conv1D, MaxPooling1D
@@ -8,20 +16,9 @@ from keras import backend as K
 from sklearn.utils import shuffle
 from sklearn.metrics import roc_auc_score
 
-import numpy as np
-import pandas as pd
-
-from ipfml import processing
-import modules.utils.config as cfg
-
-from PIL import Image
-
-import sys, os
-import argparse
-import json
+# modules and config imports
+import custom_config as cfg
 
-import subprocess
-import time
 
 def f1(y_true, y_pred):
     def recall(y_true, y_pred):