Parcourir la source

Refactoring of all display script

Jérôme BUISINE il y a 4 ans
Parent
commit
4a684e884e

+ 9 - 7
.gitignore

@@ -1,11 +1,13 @@
 # project data
 # project data
-data/*
-saved_models/*
-threshold_map/*
-models_info/*
-custom_norm/*
-learned_zones/*
-corr_indices/*
+data
+saved_models
+threshold_map
+models_info
+custom_norm
+learned_zones
+corr_indices
+results
+metric_curves
 .ipynb_checkpoints
 .ipynb_checkpoints
 
 
 # simulate_models.csv
 # simulate_models.csv

+ 3 - 0
.gitmodules

@@ -0,0 +1,3 @@
+[submodule "modules"]
+	path = modules
+	url = https://github.com/prise-3d/Thesis-CommonModules.git

+ 1 - 1
README.md

@@ -178,4 +178,4 @@ All others bash scripts are used to combine and run multiple model combinations.
 
 
 ## License
 ## License
 
 
-[The MIT license](https://github.com/prise-3d/Thesis-NoiseDetection-metrics/blob/master/LICENSE)
+[The MIT license](https://github.com/prise-3d/Thesis-NoiseDetection-attributes/blob/master/LICENSE)

+ 19 - 0
custom_config.py

@@ -0,0 +1,19 @@
+from modules.config.attributes_config import *
+
+# store all variables from global config
+context_vars = vars()
+
+# folders
+## min_max_custom_folder           = 'custom_norm'
+## correlation_indices_folder      = 'corr_indices'
+
+# variables
+## features_choices_labels         = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed', 'mscn_var_4', 'mscn_var_16', 'mscn_var_64', 'mscn_var_16_max', 'mscn_var_64_max', 'ica_diff', 'svd_trunc_diff', 'ipca_diff', 'svd_reconstruct', 'highest_sv_std_filters', 'lowest_sv_std_filters', 'highest_wave_sv_std_filters', 'lowest_wave_sv_std_filters']
+
+## models_names_list               = ["svm_model","ensemble_model","ensemble_model_v2","deep_keras"]
+## normalization_choices           = ['svd', 'svdn', 'svdne']
+
+# parameters
+## keras_epochs                    = 500
+## keras_batch                     = 32
+## val_dataset_size                = 0.2

+ 57 - 85
modules/utils/data.py

@@ -1,25 +1,26 @@
-from ipfml import processing, metrics, utils
-from modules.utils.config import *
+# main imports
+import numpy as np
+import sys
 
 
+# image transform imports
 from PIL import Image
 from PIL import Image
 from skimage import color
 from skimage import color
 from sklearn.decomposition import FastICA
 from sklearn.decomposition import FastICA
 from sklearn.decomposition import IncrementalPCA
 from sklearn.decomposition import IncrementalPCA
 from sklearn.decomposition import TruncatedSVD
 from sklearn.decomposition import TruncatedSVD
 from numpy.linalg import svd as lin_svd
 from numpy.linalg import svd as lin_svd
-
 from scipy.signal import medfilt2d, wiener, cwt
 from scipy.signal import medfilt2d, wiener, cwt
 import pywt
 import pywt
-
 import cv2
 import cv2
-import numpy as np
 
 
+from ipfml.processing import transform, compression, segmentation
+from ipfml import utils
 
 
-_scenes_names_prefix   = '_scenes_names'
-_scenes_indices_prefix = '_scenes_indices'
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
 
-# store all variables from current module context
-context_vars = vars()
+import custom_config as cfg
+from modules.utils import data as dt
 
 
 
 
 def get_svd_data(data_type, block):
 def get_svd_data(data_type, block):
@@ -31,11 +32,11 @@ def get_svd_data(data_type, block):
 
 
         block_file_path = '/tmp/lab_img.png'
         block_file_path = '/tmp/lab_img.png'
         block.save(block_file_path)
         block.save(block_file_path)
-        data = processing.get_LAB_L_SVD_s(Image.open(block_file_path))
+        data = transform.get_LAB_L_SVD_s(Image.open(block_file_path))
 
 
     if data_type == 'mscn':
     if data_type == 'mscn':
 
 
-        img_mscn_revisited = processing.rgb_to_mscn(block)
+        img_mscn_revisited = transform.rgb_to_mscn(block)
 
 
         # save tmp as img
         # save tmp as img
         img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
         img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
@@ -44,47 +45,47 @@ def get_svd_data(data_type, block):
         img_block = Image.open(mscn_revisited_file_path)
         img_block = Image.open(mscn_revisited_file_path)
 
 
         # extract from temp image
         # extract from temp image
-        data = metrics.get_SVD_s(img_block)
+        data = compression.get_SVD_s(img_block)
 
 
     """if data_type == 'mscn':
     """if data_type == 'mscn':
 
 
         img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
         img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
-        img_mscn = processing.calculate_mscn_coefficients(img_gray, 7)
-        img_mscn_norm = processing.normalize_2D_arr(img_mscn)
+        img_mscn = transform.calculate_mscn_coefficients(img_gray, 7)
+        img_mscn_norm = transform.normalize_2D_arr(img_mscn)
 
 
         img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
         img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
 
 
-        data = metrics.get_SVD_s(img_mscn_gray)
+        data = compression.get_SVD_s(img_mscn_gray)
     """
     """
 
 
     if data_type == 'low_bits_6':
     if data_type == 'low_bits_6':
 
 
-        low_bits_6 = processing.rgb_to_LAB_L_low_bits(block, 6)
-        data = metrics.get_SVD_s(low_bits_6)
+        low_bits_6 = transform.rgb_to_LAB_L_low_bits(block, 6)
+        data = compression.get_SVD_s(low_bits_6)
 
 
     if data_type == 'low_bits_5':
     if data_type == 'low_bits_5':
 
 
-        low_bits_5 = processing.rgb_to_LAB_L_low_bits(block, 5)
-        data = metrics.get_SVD_s(low_bits_5)
+        low_bits_5 = transform.rgb_to_LAB_L_low_bits(block, 5)
+        data = compression.get_SVD_s(low_bits_5)
 
 
     if data_type == 'low_bits_4':
     if data_type == 'low_bits_4':
 
 
-        low_bits_4 = processing.rgb_to_LAB_L_low_bits(block, 4)
-        data = metrics.get_SVD_s(low_bits_4)
+        low_bits_4 = transform.rgb_to_LAB_L_low_bits(block, 4)
+        data = compression.get_SVD_s(low_bits_4)
 
 
     if data_type == 'low_bits_3':
     if data_type == 'low_bits_3':
 
 
-        low_bits_3 = processing.rgb_to_LAB_L_low_bits(block, 3)
-        data = metrics.get_SVD_s(low_bits_3)
+        low_bits_3 = transform.rgb_to_LAB_L_low_bits(block, 3)
+        data = compression.get_SVD_s(low_bits_3)
 
 
     if data_type == 'low_bits_2':
     if data_type == 'low_bits_2':
 
 
-        low_bits_2 = processing.rgb_to_LAB_L_low_bits(block, 2)
-        data = metrics.get_SVD_s(low_bits_2)
+        low_bits_2 = transform.rgb_to_LAB_L_low_bits(block, 2)
+        data = compression.get_SVD_s(low_bits_2)
 
 
     if data_type == 'low_bits_4_shifted_2':
     if data_type == 'low_bits_4_shifted_2':
 
 
-        data = metrics.get_SVD_s(processing.rgb_to_LAB_L_bits(block, (3, 6)))
+        data = compression.get_SVD_s(transform.rgb_to_LAB_L_bits(block, (3, 6)))
 
 
     if data_type == 'sub_blocks_stats':
     if data_type == 'sub_blocks_stats':
 
 
@@ -92,14 +93,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         width, height, _= block.shape
         sub_width, sub_height = int(width / 4), int(height / 4)
         sub_width, sub_height = int(width / 4), int(height / 4)
 
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
 
         data = []
         data = []
 
 
         for sub_b in sub_blocks:
         for sub_b in sub_blocks:
 
 
             # by default use the whole lab L canal
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
 
 
             # get information we want from svd
             # get information we want from svd
             data.append(np.mean(l_svd_data))
             data.append(np.mean(l_svd_data))
@@ -120,14 +121,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         width, height, _= block.shape
         sub_width, sub_height = int(width / 4), int(height / 4)
         sub_width, sub_height = int(width / 4), int(height / 4)
 
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
 
         data = []
         data = []
 
 
         for sub_b in sub_blocks:
         for sub_b in sub_blocks:
 
 
             # by default use the whole lab L canal
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
 
 
             # get information we want from svd
             # get information we want from svd
             data.append(np.mean(l_svd_data))
             data.append(np.mean(l_svd_data))
@@ -145,14 +146,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         width, height, _= block.shape
         sub_width, sub_height = int(width / 8), int(height / 8)
         sub_width, sub_height = int(width / 8), int(height / 8)
 
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
 
         data = []
         data = []
 
 
         for sub_b in sub_blocks:
         for sub_b in sub_blocks:
 
 
             # by default use the whole lab L canal
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
 
 
             area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
             area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
             data.append(area_under_curve)
             data.append(area_under_curve)
@@ -166,14 +167,14 @@ def get_svd_data(data_type, block):
         width, height, _= block.shape
         width, height, _= block.shape
         sub_width, sub_height = int(width / 8), int(height / 8)
         sub_width, sub_height = int(width / 8), int(height / 8)
 
 
-        sub_blocks = processing.divide_in_blocks(block, (sub_width, sub_height))
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
 
 
         data = []
         data = []
 
 
         for sub_b in sub_blocks:
         for sub_b in sub_blocks:
 
 
             # by default use the whole lab L canal
             # by default use the whole lab L canal
-            l_svd_data = np.array(processing.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
             l_svd_data = utils.normalize_arr(l_svd_data)
             l_svd_data = utils.normalize_arr(l_svd_data)
 
 
             area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
             area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
@@ -211,7 +212,7 @@ def get_svd_data(data_type, block):
         data = data[indices]
         data = data[indices]
 
 
     if data_type == 'ica_diff':
     if data_type == 'ica_diff':
-        current_image = metrics.get_LAB_L(block)
+        current_image = transform.get_LAB_L(block)
 
 
         ica = FastICA(n_components=50)
         ica = FastICA(n_components=50)
         ica.fit(current_image)
         ica.fit(current_image)
@@ -222,14 +223,14 @@ def get_svd_data(data_type, block):
         final_image = utils.normalize_2D_arr(image_restored)
         final_image = utils.normalize_2D_arr(image_restored)
         final_image = np.array(final_image * 255, 'uint8')
         final_image = np.array(final_image * 255, 'uint8')
 
 
-        sv_values = utils.normalize_arr(metrics.get_SVD_s(current_image))
-        ica_sv_values = utils.normalize_arr(metrics.get_SVD_s(final_image))
+        sv_values = utils.normalize_arr(compression.get_SVD_s(current_image))
+        ica_sv_values = utils.normalize_arr(compression.get_SVD_s(final_image))
 
 
         data = abs(np.array(sv_values) - np.array(ica_sv_values))
         data = abs(np.array(sv_values) - np.array(ica_sv_values))
 
 
     if data_type == 'svd_trunc_diff':
     if data_type == 'svd_trunc_diff':
 
 
-        current_image = metrics.get_LAB_L(block)
+        current_image = transform.get_LAB_L(block)
 
 
         svd = TruncatedSVD(n_components=30, n_iter=100, random_state=42)
         svd = TruncatedSVD(n_components=30, n_iter=100, random_state=42)
         transformed_image = svd.fit_transform(current_image)
         transformed_image = svd.fit_transform(current_image)
@@ -237,12 +238,12 @@ def get_svd_data(data_type, block):
 
 
         reduced_image = (current_image - restored_image)
         reduced_image = (current_image - restored_image)
 
 
-        U, s, V = metrics.get_SVD(reduced_image)
+        U, s, V = compression.get_SVD(reduced_image)
         data = s
         data = s
 
 
     if data_type == 'ipca_diff':
     if data_type == 'ipca_diff':
 
 
-        current_image = metrics.get_LAB_L(block)
+        current_image = transform.get_LAB_L(block)
 
 
         transformer = IncrementalPCA(n_components=20, batch_size=25)
         transformer = IncrementalPCA(n_components=20, batch_size=25)
         transformed_image = transformer.fit_transform(current_image)
         transformed_image = transformer.fit_transform(current_image)
@@ -250,7 +251,7 @@ def get_svd_data(data_type, block):
 
 
         reduced_image = (current_image - restored_image)
         reduced_image = (current_image - restored_image)
 
 
-        U, s, V = metrics.get_SVD(reduced_image)
+        U, s, V = compression.get_SVD(reduced_image)
         data = s
         data = s
 
 
     if data_type == 'svd_reconstruct':
     if data_type == 'svd_reconstruct':
@@ -258,7 +259,7 @@ def get_svd_data(data_type, block):
         reconstructed_interval = (90, 200)
         reconstructed_interval = (90, 200)
         begin, end = reconstructed_interval
         begin, end = reconstructed_interval
 
 
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         lab_img = np.array(lab_img, 'uint8')
         lab_img = np.array(lab_img, 'uint8')
 
 
         U, s, V = lin_svd(lab_img, full_matrices=True)
         U, s, V = lin_svd(lab_img, full_matrices=True)
@@ -269,12 +270,12 @@ def get_svd_data(data_type, block):
 
 
         output_img = np.array(output_img, 'uint8')
         output_img = np.array(output_img, 'uint8')
 
 
-        data = metrics.get_SVD_s(output_img)
+        data = compression.get_SVD_s(output_img)
 
 
     if 'sv_std_filters' in data_type:
     if 'sv_std_filters' in data_type:
 
 
         # convert into lab by default to apply filters
         # convert into lab by default to apply filters
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         arr = np.array(lab_img)
         arr = np.array(lab_img)
         images = []
         images = []
         
         
@@ -285,12 +286,12 @@ def get_svd_data(data_type, block):
         images.append(wiener(arr, [5, 5]))
         images.append(wiener(arr, [5, 5]))
         
         
         # By default computation of current block image
         # By default computation of current block image
-        s_arr = metrics.get_SVD_s(arr)
+        s_arr = compression.get_SVD_s(arr)
         sv_vector = [s_arr]
         sv_vector = [s_arr]
 
 
         # for each new image apply SVD and get SV 
         # for each new image apply SVD and get SV 
         for img in images:
         for img in images:
-            s = metrics.get_SVD_s(img)
+            s = compression.get_SVD_s(img)
             sv_vector.append(s)
             sv_vector.append(s)
             
             
         sv_array = np.array(sv_vector)
         sv_array = np.array(sv_vector)
@@ -307,10 +308,10 @@ def get_svd_data(data_type, block):
         indices = []
         indices = []
 
 
         if 'lowest' in data_type:
         if 'lowest' in data_type:
-            indices = get_lowest_values(sv_std, 200)
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
 
 
         if 'highest' in data_type:
         if 'highest' in data_type:
-            indices = get_highest_values(sv_std, 200)
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
 
 
         # data are arranged following std trend computed
         # data are arranged following std trend computed
         data = s_arr[indices]
         data = s_arr[indices]
@@ -319,7 +320,7 @@ def get_svd_data(data_type, block):
     if 'wave_sv_std_filters' in data_type:
     if 'wave_sv_std_filters' in data_type:
 
 
         # convert into lab by default to apply filters
         # convert into lab by default to apply filters
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         arr = np.array(lab_img)
         arr = np.array(lab_img)
         images = []
         images = []
         
         
@@ -335,12 +336,12 @@ def get_svd_data(data_type, block):
         images.append(w2d(arr, 'haar', 4))
         images.append(w2d(arr, 'haar', 4))
         
         
         # By default computation of current block image
         # By default computation of current block image
-        s_arr = metrics.get_SVD_s(arr)
+        s_arr = compression.get_SVD_s(arr)
         sv_vector = [s_arr]
         sv_vector = [s_arr]
 
 
         # for each new image apply SVD and get SV 
         # for each new image apply SVD and get SV 
         for img in images:
         for img in images:
-            s = metrics.get_SVD_s(img)
+            s = compression.get_SVD_s(img)
             sv_vector.append(s)
             sv_vector.append(s)
             
             
         sv_array = np.array(sv_vector)
         sv_array = np.array(sv_vector)
@@ -357,10 +358,10 @@ def get_svd_data(data_type, block):
         indices = []
         indices = []
 
 
         if 'lowest' in data_type:
         if 'lowest' in data_type:
-            indices = get_lowest_values(sv_std, 200)
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
 
 
         if 'highest' in data_type:
         if 'highest' in data_type:
-            indices = get_highest_values(sv_std, 200)
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
 
 
         # data are arranged following std trend computed
         # data are arranged following std trend computed
         data = s_arr[indices]
         data = s_arr[indices]
@@ -369,7 +370,7 @@ def get_svd_data(data_type, block):
 
 
         img_width, img_height = 200, 200
         img_width, img_height = 200, 200
 
 
-        lab_img = metrics.get_LAB_L(block)
+        lab_img = transform.get_LAB_L(block)
         arr = np.array(lab_img)
         arr = np.array(lab_img)
 
 
         # compute all filters statistics
         # compute all filters statistics
@@ -429,14 +430,6 @@ def get_svd_data(data_type, block):
     return data
     return data
 
 
 
 
-def get_highest_values(arr, n):
-    return np.array(arr).argsort()[-n:][::-1]
-
-
-def get_lowest_values(arr, n):
-    return np.array(arr).argsort()[::-1][-n:][::-1]
-
-
 def w2d(arr, mode='haar', level=1):
 def w2d(arr, mode='haar', level=1):
     #convert to float   
     #convert to float   
     imArray = arr
     imArray = arr
@@ -458,35 +451,14 @@ def w2d(arr, mode='haar', level=1):
 
 
 def _get_mscn_variance(block, sub_block_size=(50, 50)):
 def _get_mscn_variance(block, sub_block_size=(50, 50)):
 
 
-    blocks = processing.divide_in_blocks(block, sub_block_size)
+    blocks = segmentation.divide_in_blocks(block, sub_block_size)
 
 
     data = []
     data = []
 
 
     for block in blocks:
     for block in blocks:
-        mscn_coefficients = processing.get_mscn_coefficients(block)
+        mscn_coefficients = transform.get_mscn_coefficients(block)
         flat_coeff = mscn_coefficients.flatten()
         flat_coeff = mscn_coefficients.flatten()
         data.append(np.var(flat_coeff))
         data.append(np.var(flat_coeff))
 
 
     return np.sort(data)
     return np.sort(data)
 
 
-
-def get_renderer_scenes_indices(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_indices
-    else:
-        return context_vars[renderer_name + _scenes_indices_prefix]
-
-def get_renderer_scenes_names(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_names
-    else:
-        return context_vars[renderer_name + _scenes_names_prefix]
-

+ 1 - 0
dataset

@@ -0,0 +1 @@
+../data/Scenes/

+ 32 - 50
display_bits_shifted_scene.py

@@ -1,40 +1,34 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
 import numpy as np
 import numpy as np
 import random
 import random
 import time
 import time
 import json
 import json
 
 
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing
-from ipfml import metrics
 from skimage import color
 from skimage import color
 import matplotlib.pyplot as plt
 import matplotlib.pyplot as plt
 
 
-from modules.utils import config as cfg
+from ipfml.processing import compression, transform
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
 
+import custom_config as cfg
+from modules.utils import data as dt
 
 
-config_filename     = cfg.config_filename
+# variables and parameters
 zone_folder         = cfg.zone_folder
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 min_max_filename    = cfg.min_max_filename_extension
 
 
 # define all scenes values
 # define all scenes values
 scenes_list         = cfg.scenes_names
 scenes_list         = cfg.scenes_names
 scenes_indices      = cfg.scenes_indices
 scenes_indices      = cfg.scenes_indices
-choices             = cfg.normalization_choices
 path                = cfg.dataset_path
 path                = cfg.dataset_path
 zones               = cfg.zones_indices
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 seuil_expe_filename = cfg.seuil_expe_filename
 
 
-metric_choices      = cfg.metric_choices_labels
 max_nb_bits = 8
 max_nb_bits = 8
 
 
 def display_data_scenes(nb_bits, p_scene):
 def display_data_scenes(nb_bits, p_scene):
@@ -50,21 +44,12 @@ def display_data_scenes(nb_bits, p_scene):
     scenes = [s for s in scenes if min_max_filename not in s]
     scenes = [s for s in scenes if min_max_filename not in s]
 
 
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         if p_scene == folder_scene:
         if p_scene == folder_scene:
             print(folder_scene)
             print(folder_scene)
             scene_path = os.path.join(path, folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             # construct each zones folder name
             zones_folder = []
             zones_folder = []
 
 
@@ -77,16 +62,12 @@ def display_data_scenes(nb_bits, p_scene):
                 current_zone = "zone"+index_str
                 current_zone = "zone"+index_str
                 zones_folder.append(current_zone)
                 zones_folder.append(current_zone)
 
 
-            zones_images_data = []
             threshold_info = []
             threshold_info = []
 
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
 
 
                 zone_path = os.path.join(scene_path, zone_folder)
                 zone_path = os.path.join(scene_path, zone_folder)
 
 
-                current_counter_index = int(start_index_image)
-                end_counter_index = int(end_index_image)
-
                 # get threshold information
                 # get threshold information
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
 
@@ -101,25 +82,28 @@ def display_data_scenes(nb_bits, p_scene):
             print(mean_threshold, "mean threshold found")
             print(mean_threshold, "mean threshold found")
             threshold_image_found = False
             threshold_image_found = False
 
 
-            # find appropriate mean threshold picture
-            while(current_counter_index <= end_counter_index and not threshold_image_found):
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_image_path = scene_images[0]
+            end_image_path   = scene_images[-1]
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
 
 
-                if mean_threshold < int(current_counter_index):
-                    current_counter_index_str = str(current_counter_index)
+            # for each images
+            for img_path in scene_images:
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
 
-                    while len(start_index_image) > len(current_counter_index_str):
-                        current_counter_index_str = "0" + current_counter_index_str
+                if mean_threshold < int(current_quality_image) and not threshold_image_found:
 
 
                     threshold_image_found = True
                     threshold_image_found = True
-                    threshold_image_zone = current_counter_index_str
+                    threshold_image_path = img_path
 
 
-                current_counter_index += step_counter
+                    threshold_image = dt.get_scene_image_quality(img_path)
 
 
             # all indexes of picture to plot
             # all indexes of picture to plot
-            images_indexes = [start_index_image, threshold_image_zone, end_index_image]
-            images_data = []
-
-            print(images_indexes)
+            images_path = [start_image_path, threshold_image_path, end_image_path]
 
 
             low_bits_svd_values = []
             low_bits_svd_values = []
 
 
@@ -127,16 +111,14 @@ def display_data_scenes(nb_bits, p_scene):
 
 
                 low_bits_svd_values.append([])
                 low_bits_svd_values.append([])
 
 
-                for index in images_indexes:
-
-                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
+                for img_path in images_path:
 
 
                     current_img = Image.open(img_path)
                     current_img = Image.open(img_path)
 
 
                     block_used = np.array(current_img)
                     block_used = np.array(current_img)
 
 
-                    low_bits_block = processing.rgb_to_LAB_L_bits(block_used, (i + 1, i + nb_bits + 1))
-                    low_bits_svd = metrics.get_SVD_s(low_bits_block)
+                    low_bits_block = transform.rgb_to_LAB_L_bits(block_used, (i + 1, i + nb_bits + 1))
+                    low_bits_svd = compression.get_SVD_s(low_bits_block)
                     low_bits_svd = [b / low_bits_svd[0] for b in low_bits_svd]
                     low_bits_svd = [b / low_bits_svd[0] for b in low_bits_svd]
                     low_bits_svd_values[i].append(low_bits_svd)
                     low_bits_svd_values[i].append(low_bits_svd)
 
 
@@ -146,9 +128,9 @@ def display_data_scenes(nb_bits, p_scene):
 
 
             for id, data in enumerate(low_bits_svd_values):
             for id, data in enumerate(low_bits_svd_values):
                 fig.add_subplot(3, 3, (id + 1))
                 fig.add_subplot(3, 3, (id + 1))
-                plt.plot(data[0], label='Noisy_' + start_index_image)
-                plt.plot(data[1], label='Threshold_' + threshold_image_zone)
-                plt.plot(data[2], label='Reference_' + end_index_image)
+                plt.plot(data[0], label='Noisy_' + start_quality_image)
+                plt.plot(data[1], label='Threshold_' + threshold_image)
+                plt.plot(data[2], label='Reference_' + end_quality_image)
                 plt.ylabel('Lab SVD ' + str(nb_bits) + ' bits values shifted by ' + str(id), fontsize=14)
                 plt.ylabel('Lab SVD ' + str(nb_bits) + ' bits values shifted by ' + str(id), fontsize=14)
                 plt.xlabel('Vector features', fontsize=16)
                 plt.xlabel('Vector features', fontsize=16)
                 plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)
                 plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)

+ 176 - 0
display/display_scenes_zones.py

@@ -0,0 +1,176 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+import random
+import time
+import json
+
+# image processing imports
+from PIL import Image
+from skimage import color
+import matplotlib.pyplot as plt
+
+from data_attributes import get_svd_data
+
+from ipfml.processing import segmentation, transform, compression
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
+
+# variables and parameters
+zone_folder         = cfg.zone_folder
+min_max_filename    = cfg.min_max_filename_extension
+
+# define all scenes values
+scenes_list         = cfg.scenes_names
+scenes_indices      = cfg.scenes_indices
+norm_choices        = cfg.normalization_choices
+path                = cfg.dataset_path
+zones               = cfg.zones_indices
+seuil_expe_filename = cfg.seuil_expe_filename
+
+features_choices      = cfg.features_choices_labels
+
+
+def display_data_scenes(data_type, p_scene, p_kind):
+    """
+    @brief Method which displays data from scene
+    @param data_type,  feature choice
+    @param scene, scene choice
+    @param mode, normalization choice
+    @return nothing
+    """
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    # go ahead each scenes
+    for folder_scene in scenes:
+
+        if p_scene == folder_scene:
+            print(folder_scene)
+            scene_path = os.path.join(path, folder_scene)
+
+            # construct each zones folder name
+            zones_folder = []
+
+            # get zones list info
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+
+                current_zone = "zone"+index_str
+                zones_folder.append(current_zone)
+
+            zones_images_data = []
+            threshold_info = []
+
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_image_path = scene_images[0]
+            end_image_path   = scene_images[-1]
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
+
+            for id_zone, zone_folder in enumerate(zones_folder):
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                # get threshold information
+                path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+                # open treshold path and get this information
+                with open(path_seuil, "r") as seuil_file:
+                    threshold_learned = int(seuil_file.readline().strip())
+
+                threshold_image_found = False
+
+                for img_path in scene_images:
+                    current_quality_image = dt.get_scene_image_quality(img_path)
+
+                    if threshold_learned < int(current_quality_image) and not threshold_image_found:
+
+                        threshold_image_found = True
+                        threshold_image_path = img_path
+
+                        threshold_image = dt.get_scene_image_postfix(img_path)
+                        threshold_info.append(threshold_image)
+
+                # all indexes of picture to plot
+                images_path = [start_image_path, threshold_image_path, end_image_path]
+                images_data = []
+
+                for img_path in images_path:
+
+                    current_img = Image.open(img_path)
+                    img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
+
+                    # getting expected block id
+                    block = img_blocks[id_zone]
+
+                    data = get_svd_data(data_type, block)
+
+                    ##################
+                    # Data mode part #
+                    ##################
+
+                    # modify data depending mode
+
+                    if p_kind == 'svdn':
+                        data = utils.normalize_arr(data)
+
+                    if p_kind == 'svdne':
+                        path_min_max = os.path.join(path, data_type + min_max_filename)
+
+                        with open(path_min_max, 'r') as f:
+                            min_val = float(f.readline())
+                            max_val = float(f.readline())
+
+                        data = utils.normalize_arr_with_range(data, min_val, max_val)
+
+                    # append of data
+                    images_data.append(data)
+
+                zones_images_data.append(images_data)
+
+            fig=plt.figure(figsize=(8, 8))
+            fig.suptitle(data_type + " values for " + p_scene + " scene (normalization : " + p_kind + ")", fontsize=20)
+
+            for id, data in enumerate(zones_images_data):
+                fig.add_subplot(4, 4, (id + 1))
+                plt.plot(data[0], label='Noisy_' + start_quality_image)
+                plt.plot(data[1], label='Threshold_' + threshold_info[id])
+                plt.plot(data[2], label='Reference_' + end_quality_image)
+                plt.ylabel(data_type + ' SVD, ZONE_' + str(id + 1), fontsize=18)
+                plt.xlabel('Vector features', fontsize=18)
+                plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=18)
+                plt.ylim(0, 0.1)
+            plt.show()
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Display zones curves of feature on scene ")
+
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
+    parser.add_argument('--scene', type=str, help='scene index to use', choices=scenes_indices)
+    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=norm_choices)
+
+    args = parser.parse_args()
+
+    p_feature = args.feature
+    p_kind   = args.kind
+    p_scene  = scenes_list[scenes_indices.index(args.scene)]
+
+    display_data_scenes(p_feature, p_scene, p_kind)
+
+if __name__== "__main__":
+    main()

+ 37 - 48
display_scenes_zones_shifted.py

@@ -1,39 +1,36 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
 import numpy as np
 import numpy as np
 import random
 import random
 import time
 import time
 import json
 import json
 
 
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing, metrics, utils
 from skimage import color
 from skimage import color
 import matplotlib.pyplot as plt
 import matplotlib.pyplot as plt
 
 
-from modules.utils import config as cfg
+from ipfml.processing import segmentation, transform, compression
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 
-config_filename     = cfg.config_filename
+
+# variables and parameters
 zone_folder         = cfg.zone_folder
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 min_max_filename    = cfg.min_max_filename_extension
 
 
 # define all scenes values
 # define all scenes values
 scenes_list         = cfg.scenes_names
 scenes_list         = cfg.scenes_names
 scenes_indices      = cfg.scenes_indices
 scenes_indices      = cfg.scenes_indices
-choices             = cfg.normalization_choices
 path                = cfg.dataset_path
 path                = cfg.dataset_path
 zones               = cfg.zones_indices
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 seuil_expe_filename = cfg.seuil_expe_filename
 
 
-metric_choices      = cfg.metric_choices_labels
-
 max_nb_bits = 8
 max_nb_bits = 8
 
 
 def display_data_scenes(p_scene, p_bits, p_shifted):
 def display_data_scenes(p_scene, p_bits, p_shifted):
@@ -50,21 +47,12 @@ def display_data_scenes(p_scene, p_bits, p_shifted):
     scenes = [s for s in scenes if min_max_filename not in s]
     scenes = [s for s in scenes if min_max_filename not in s]
 
 
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         if p_scene == folder_scene:
         if p_scene == folder_scene:
             print(folder_scene)
             print(folder_scene)
             scene_path = os.path.join(path, folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             # construct each zones folder name
             zones_folder = []
             zones_folder = []
 
 
@@ -80,55 +68,56 @@ def display_data_scenes(p_scene, p_bits, p_shifted):
             zones_images_data = []
             zones_images_data = []
             threshold_info = []
             threshold_info = []
 
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+
+            start_image_path = scene_images[0]
+            end_image_path   = scene_images[-1]
+
+            start_quality_image = dt.get_scene_image_quality(scene_images[0])
+            end_quality_image   = dt.get_scene_image_quality(scene_images[-1])
+
             for id_zone, zone_folder in enumerate(zones_folder):
             for id_zone, zone_folder in enumerate(zones_folder):
 
 
                 zone_path = os.path.join(scene_path, zone_folder)
                 zone_path = os.path.join(scene_path, zone_folder)
 
 
-                current_counter_index = int(start_index_image)
-                end_counter_index = int(end_index_image)
-
                 # get threshold information
                 # get threshold information
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
 
                 # open treshold path and get this information
                 # open treshold path and get this information
                 with open(path_seuil, "r") as seuil_file:
                 with open(path_seuil, "r") as seuil_file:
-                    seuil_learned = int(seuil_file.readline().strip())
+                    threshold_learned = int(seuil_file.readline().strip())
 
 
                 threshold_image_found = False
                 threshold_image_found = False
-                while(current_counter_index <= end_counter_index and not threshold_image_found):
 
 
-                    if seuil_learned < int(current_counter_index):
-                        current_counter_index_str = str(current_counter_index)
+                # for each images
+                for img_path in scene_images:
+                    current_quality_image = dt.get_scene_image_quality(img_path)
 
 
-                        while len(start_index_image) > len(current_counter_index_str):
-                            current_counter_index_str = "0" + current_counter_index_str
+                    if threshold_learned < int(current_quality_image) and not threshold_image_found:
 
 
                         threshold_image_found = True
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
-                        threshold_info.append(threshold_image_zone)
+                        threshold_image_path = img_path
 
 
-                    current_counter_index += step_counter
+                        threshold_image = dt.get_scene_image_postfix(img_path)
+                        threshold_info.append(threshold_image)
 
 
                 # all indexes of picture to plot
                 # all indexes of picture to plot
-                images_indexes = [start_index_image, threshold_image_zone, end_index_image]
+                images_path = [start_image_path, threshold_image_path, end_image_path]
                 images_data = []
                 images_data = []
 
 
-                print(images_indexes)
-
-                for index in images_indexes:
-
-                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
+                for img_path in images_path:
 
 
                     current_img = Image.open(img_path)
                     current_img = Image.open(img_path)
-                    img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+                    img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
 
                     # getting expected block id
                     # getting expected block id
                     block = img_blocks[id_zone]
                     block = img_blocks[id_zone]
 
 
                     # get data from mode
                     # get data from mode
                     # Here you can add the way you compute data
                     # Here you can add the way you compute data
-                    low_bits_block = processing.rgb_to_LAB_L_bits(block, (p_shifted + 1, p_shifted + p_bits + 1))
-                    data = metrics.get_SVD_s(low_bits_block)
+                    low_bits_block = transform.rgb_to_LAB_L_bits(block, (p_shifted + 1, p_shifted + p_bits + 1))
+                    data = compression.get_SVD_s(low_bits_block)
 
 
                     ##################
                     ##################
                     # Data mode part #
                     # Data mode part #
@@ -145,9 +134,9 @@ def display_data_scenes(p_scene, p_bits, p_shifted):
 
 
             for id, data in enumerate(zones_images_data):
             for id, data in enumerate(zones_images_data):
                 fig.add_subplot(4, 4, (id + 1))
                 fig.add_subplot(4, 4, (id + 1))
-                plt.plot(data[0], label='Noisy_' + start_index_image)
+                plt.plot(data[0], label='Noisy_' + start_quality_image)
                 plt.plot(data[1], label='Threshold_' + threshold_info[id])
                 plt.plot(data[1], label='Threshold_' + threshold_info[id])
-                plt.plot(data[2], label='Reference_' + end_index_image)
+                plt.plot(data[2], label='Reference_' + end_quality_image)
                 plt.ylabel('Lab SVD ' + str(p_bits) + ' bits shifted by ' + str(p_shifted) + ', ZONE_' + str(id + 1), fontsize=14)
                 plt.ylabel('Lab SVD ' + str(p_bits) + ' bits shifted by ' + str(p_shifted) + ', ZONE_' + str(id + 1), fontsize=14)
                 plt.xlabel('Vector features', fontsize=16)
                 plt.xlabel('Vector features', fontsize=16)
                 plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)
                 plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=14)

+ 10 - 4
display_simulation_curves.py

@@ -1,13 +1,19 @@
+# main imports
 import numpy as np
 import numpy as np
 import pandas as pd
 import pandas as pd
+import os, sys, argparse
 
 
+# image processing imports
 import matplotlib.pyplot as plt
 import matplotlib.pyplot as plt
-import os, sys, argparse
+from data_attributes import get_svd_data
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
 
-from modules.utils.data import get_svd_data
+import custom_config as cfg
 
 
-from modules.utils import config as cfg
 
 
+# variables and parameters
 learned_zones_folder = cfg.learned_zones_folder
 learned_zones_folder = cfg.learned_zones_folder
 models_name          = cfg.models_names_list
 models_name          = cfg.models_names_list
 label_freq           = 6
 label_freq           = 6
@@ -25,7 +31,7 @@ def display_curves(folder_path, model_name):
             data_filename = model_name
             data_filename = model_name
             learned_zones_folder_path = os.path.join(learned_zones_folder, data_filename)
             learned_zones_folder_path = os.path.join(learned_zones_folder, data_filename)
 
 
-    data_files = [x for x in os.listdir(folder_path) if '.png' not in x]
+    data_files = [x for x in os.listdir(folder_path) if cfg.scene_image_extension not in x]
 
 
     scene_names = [f.split('_')[3] for f in data_files]
     scene_names = [f.split('_')[3] for f in data_files]
 
 

+ 34 - 60
display_svd_area_data_scene.py

@@ -1,32 +1,25 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
-
 import numpy as np
 import numpy as np
-import random
-import time
-import json
 
 
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
-
 from skimage import color
 from skimage import color
-
 import matplotlib.pyplot as plt
 import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
 
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+from ipfml.processing import segmentation, transform, compression
+from ipfml import utils
+import ipfml.iqa.fr as fr_iqa
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 
 # getting configuration information
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 min_max_filename    = cfg.min_max_filename_extension
 
 
@@ -38,7 +31,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 seuil_expe_filename = cfg.seuil_expe_filename
 
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 
 max_nb_bits = 8
 max_nb_bits = 8
 
 
@@ -46,7 +39,6 @@ integral_area_choices = ['trapz', 'simps']
 
 
 def get_area_under_curve(p_area, p_data):
 def get_area_under_curve(p_area, p_data):
 
 
-    noise_method = None
     function_name = 'integral_area_' + p_area
     function_name = 'integral_area_' + p_area
 
 
     try:
     try:
@@ -74,8 +66,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     max_value_svd = 0
     max_value_svd = 0
     min_value_svd = sys.maxsize
     min_value_svd = sys.maxsize
 
 
-    image_indices = []
-
     scenes = os.listdir(path)
     scenes = os.listdir(path)
     # remove min max file from scenes folder
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
     scenes = [s for s in scenes if min_max_filename not in s]
@@ -83,23 +73,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     begin_data, end_data = p_interval
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
     begin_index, end_index = p_indices
 
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         if p_scene == folder_scene:
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             # construct each zones folder name
             zones_folder = []
             zones_folder = []
 
 
@@ -117,6 +96,10 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
 
             threshold_learned_zones = []
             threshold_learned_zones = []
 
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+            number_scene_image = len(scene_images)
+
             for id, zone_folder in enumerate(zones_folder):
             for id, zone_folder in enumerate(zones_folder):
 
 
                 # get threshold information
                 # get threshold information
@@ -128,25 +111,17 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned_zones.append(threshold_learned)
                     threshold_learned_zones.append(threshold_learned)
 
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_image_found = False
             threshold_image_found = False
 
 
-            file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
             svd_data = []
             svd_data = []
 
 
-            while(current_counter_index <= end_counter_index):
-
-                current_counter_index_str = str(current_counter_index)
-
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+            # for each images
+            for id_img, img_path in enumerate(scene_images):
+                
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
 
-                image_path = file_path.format(str(current_counter_index_str))
-                img = Image.open(image_path)
+                img = Image.open(img_path)
 
 
                 svd_values = get_svd_data(p_metric, img)
                 svd_values = get_svd_data(p_metric, img)
 
 
@@ -164,25 +139,24 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     max_value_svd = max_value
                     max_value_svd = max_value
 
 
                 # keep in memory used data
                 # keep in memory used data
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+                        images_indices.append(current_quality_image)
                         svd_data.append(svd_values)
                         svd_data.append(svd_values)
 
 
-                    if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                    if threshold_mean < current_quality_image and not threshold_image_found:
 
 
                         threshold_image_found = True
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
+                        image_name_postfix = dt.get_scene_image_postfix(img_path)
+                        threshold_image_zone = image_name_postfix
 
 
-                current_counter_index += step_counter
-                print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+                print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
                 sys.stdout.write("\033[F")
                 sys.stdout.write("\033[F")
 
 
 
 
             # all indices of picture to plot
             # all indices of picture to plot
             print(images_indices)
             print(images_indices)
 
 
-            previous_data = []
             area_data = []
             area_data = []
 
 
             for id, data in enumerate(svd_data):
             for id, data in enumerate(svd_data):
@@ -249,7 +223,7 @@ def main():
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='Feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -261,14 +235,14 @@ def main():
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_mode     = args.mode
     p_mode     = args.mode
     p_step     = args.step
     p_step     = args.step
     p_norm     = args.norm
     p_norm     = args.norm
     p_area     = args.area
     p_area     = args.area
     p_ylim     = list(map(int, args.ylim.split(',')))
     p_ylim     = list(map(int, args.ylim.split(',')))
 
 
-    display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_area, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_area, p_ylim)
 
 
 if __name__== "__main__":
 if __name__== "__main__":
     main()
     main()

+ 36 - 67
display_svd_area_scenes.py

@@ -1,32 +1,23 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
-
 import numpy as np
 import numpy as np
-import random
-import time
-import json
 
 
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
+import matplotlib.pyplot as plt
 
 
-from skimage import color
+from data_attributes import get_svd_data
 
 
-import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
 
 
-from modules.utils import config as cfg
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 
 # getting configuration information
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 min_max_filename    = cfg.min_max_filename_extension
 
 
@@ -38,7 +29,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 seuil_expe_filename = cfg.seuil_expe_filename
 
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 
 max_nb_bits = 8
 max_nb_bits = 8
 
 
@@ -46,7 +37,6 @@ integral_area_choices = ['trapz', 'simps']
 
 
 def get_area_under_curve(p_area, p_data):
 def get_area_under_curve(p_area, p_data):
 
 
-    noise_method = None
     function_name = 'integral_area_' + p_area
     function_name = 'integral_area_' + p_area
 
 
     try:
     try:
@@ -60,10 +50,9 @@ def get_area_under_curve(p_area, p_data):
 def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_area, p_ylim):
 def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_area, p_ylim):
     """
     """
     @brief Method which gives information about svd curves from zone of picture
     @brief Method which gives information about svd curves from zone of picture
-    @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of svd data to display
-    @param p_interval, interval [begin, end] of samples or minutes from render generation engine
-    @param p_metric, metric computed to show
+    @param p_indices, indices to display
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_mode, normalization's mode
     @param p_norm, normalization or not of selected svd data
     @param p_norm, normalization or not of selected svd data
     @param p_area, area method name to compute area under curve
     @param p_area, area method name to compute area under curve
@@ -80,30 +69,19 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
     begin_data, end_data = p_interval
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
     begin_index, end_index = p_indices
 
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # Store all informations about scenes
     # Store all informations about scenes
     scenes_area_data = []
     scenes_area_data = []
     scenes_images_indices = []
     scenes_images_indices = []
     scenes_threshold_mean = []
     scenes_threshold_mean = []
 
 
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         max_value_svd = 0
         max_value_svd = 0
         min_value_svd = sys.maxsize
         min_value_svd = sys.maxsize
 
 
         scene_path = os.path.join(path, folder_scene)
         scene_path = os.path.join(path, folder_scene)
 
 
-        config_file_path = os.path.join(scene_path, config_filename)
-
-        with open(config_file_path, "r") as config_file:
-            last_image_name = config_file.readline().strip()
-            prefix_image_name = config_file.readline().strip()
-            start_index_image = config_file.readline().strip()
-            end_index_image = config_file.readline().strip()
-            step_counter = int(config_file.readline().strip())
-
         # construct each zones folder name
         # construct each zones folder name
         zones_folder = []
         zones_folder = []
 
 
@@ -121,6 +99,10 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
         images_indices = []
         images_indices = []
         threshold_learned_zones = []
         threshold_learned_zones = []
 
 
+        # get all images of folder
+        scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+        number_scene_image = len(scene_images)
+
         for id, zone_folder in enumerate(zones_folder):
         for id, zone_folder in enumerate(zones_folder):
 
 
             # get threshold information
             # get threshold information
@@ -132,26 +114,18 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
                 threshold_learned = int(seuil_file.readline().strip())
                 threshold_learned = int(seuil_file.readline().strip())
                 threshold_learned_zones.append(threshold_learned)
                 threshold_learned_zones.append(threshold_learned)
 
 
-        current_counter_index = int(start_index_image)
-        end_counter_index = int(end_index_image)
-
         threshold_mean = np.mean(np.asarray(threshold_learned_zones))
         threshold_mean = np.mean(np.asarray(threshold_learned_zones))
         threshold_image_found = False
         threshold_image_found = False
         scenes_threshold_mean.append(int(threshold_mean / p_step))
         scenes_threshold_mean.append(int(threshold_mean / p_step))
 
 
-        file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
         svd_data = []
         svd_data = []
 
 
-        while(current_counter_index <= end_counter_index):
-
-            current_counter_index_str = str(current_counter_index)
+        # for each images
+        for id_img, img_path in enumerate(scene_images):
+            
+            current_quality_image = dt.get_scene_image_quality(img_path)
 
 
-            while len(start_index_image) > len(current_counter_index_str):
-                current_counter_index_str = "0" + current_counter_index_str
-
-            image_path = file_path.format(str(current_counter_index_str))
-            img = Image.open(image_path)
+            img = Image.open(img_path)
 
 
             svd_values = get_svd_data(p_metric, img)
             svd_values = get_svd_data(p_metric, img)
 
 
@@ -169,25 +143,22 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
                 max_value_svd = max_value
                 max_value_svd = max_value
 
 
             # keep in memory used data
             # keep in memory used data
-            if current_counter_index % p_step == 0:
-                if current_counter_index >= begin_index and current_counter_index <= end_index:
-                    images_indices.append(current_counter_index_str)
+            if current_quality_image % p_step == 0:
+                if current_quality_image >= begin_index and current_quality_image <= end_index:
+                    images_indices.append(dt.get_scene_image_postfix(img_path))
                     svd_data.append(svd_values)
                     svd_data.append(svd_values)
 
 
-                if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                if threshold_mean < current_quality_image and not threshold_image_found:
 
 
                     threshold_image_found = True
                     threshold_image_found = True
-                    threshold_image_zone = current_counter_index_str
 
 
-            current_counter_index += step_counter
-            print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+            print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
             sys.stdout.write("\033[F")
             sys.stdout.write("\033[F")
 
 
 
 
             # all indices of picture to plot
             # all indices of picture to plot
         print("Scene %s : %s" % (folder_scene, images_indices))
         print("Scene %s : %s" % (folder_scene, images_indices))
 
 
-
         scenes_images_indices.append(image_indices)
         scenes_images_indices.append(image_indices)
 
 
         area_data = []
         area_data = []
@@ -225,14 +196,12 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
         threshold_id = 0
         threshold_id = 0
         scene_name = scenes[id]
         scene_name = scenes[id]
         image_indices = scenes_images_indices[id]
         image_indices = scenes_images_indices[id]
-        threshold_image_zone = scenes_threshold_mean[id]
 
 
         p_label = scene_name + '_' + str(images_indices[id])
         p_label = scene_name + '_' + str(images_indices[id])
 
 
         threshold_id = scenes_threshold_mean[id]
         threshold_id = scenes_threshold_mean[id]
 
 
         print(p_label)
         print(p_label)
-        start_ylim, end_ylim = p_ylim
 
 
         plt.plot(area_data, label=p_label)
         plt.plot(area_data, label=p_label)
         #ax2.set_xticks(range(len(images_indices)))
         #ax2.set_xticks(range(len(images_indices)))
@@ -242,8 +211,8 @@ def display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm,
             plt.plot([threshold_id, threshold_id], [np.min(area_data), np.max(area_data)], 'k-', lw=2, color='red')
             plt.plot([threshold_id, threshold_id], [np.min(area_data), np.max(area_data)], 'k-', lw=2, color='red')
 
 
 
 
-    #start_ylim, end_ylim = p_ylim
-    #plt.ylim(start_ylim, end_ylim)
+    start_ylim, end_ylim = p_ylim
+    plt.ylim(start_ylim, end_ylim)
 
 
     plt.show()
     plt.show()
 
 
@@ -251,10 +220,10 @@ def main():
 
 
     parser = argparse.ArgumentParser(description="Display area under curve on scene")
     parser = argparse.ArgumentParser(description="Display area under curve on scene")
 
 
-    parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
+    #parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='Metric data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -263,17 +232,17 @@ def main():
 
 
     args = parser.parse_args()
     args = parser.parse_args()
 
 
-    p_scene    = scenes_list[scenes_indices.index(args.scene)]
+    #p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_mode     = args.mode
     p_mode     = args.mode
     p_step     = args.step
     p_step     = args.step
     p_norm     = args.norm
     p_norm     = args.norm
     p_area     = args.area
     p_area     = args.area
     p_ylim     = list(map(int, args.ylim.split(',')))
     p_ylim     = list(map(int, args.ylim.split(',')))
 
 
-    display_svd_values(p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_area, p_ylim)
+    display_svd_values(p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_area, p_ylim)
 
 
 if __name__== "__main__":
 if __name__== "__main__":
     main()
     main()

+ 49 - 77
display_svd_data_error_scene.py

@@ -1,32 +1,24 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
-
 import numpy as np
 import numpy as np
-import random
-import time
-import json
 
 
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
-
 from skimage import color
 from skimage import color
-
 import matplotlib.pyplot as plt
 import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
 
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 
 # getting configuration information
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 min_max_filename    = cfg.min_max_filename_extension
 
 
@@ -38,7 +30,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 seuil_expe_filename = cfg.seuil_expe_filename
 
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 
 max_nb_bits         = 8
 max_nb_bits         = 8
 display_error       = False
 display_error       = False
@@ -48,7 +40,6 @@ error_data_choices  = ['mae', 'mse', 'ssim', 'psnr']
 
 
 def get_error_distance(p_error, y_true, y_test):
 def get_error_distance(p_error, y_true, y_test):
 
 
-    noise_method = None
     function_name = p_error
     function_name = p_error
 
 
     try:
     try:
@@ -59,16 +50,16 @@ def get_error_distance(p_error, y_true, y_test):
     return error_method(y_true, y_test)
     return error_method(y_true, y_test)
 
 
 
 
-def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_error, p_ylim):
+def display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_error, p_ylim):
     """
     """
     @brief Method which gives information about svd curves from zone of picture
     @brief Method which gives information about svd curves from zone of picture
     @param p_scene, scene expected to show svd values
     @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
-    @param p_metric, metric computed to show
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_mode, normalization's mode
     @param p_norm, normalization or not of selected svd data
     @param p_norm, normalization or not of selected svd data
-    @param p_error, error metric used to display
+    @param p_error, error feature used to display
     @param p_ylim, ylim choice to better display of data
     @param p_ylim, ylim choice to better display of data
     @return nothing
     @return nothing
     """
     """
@@ -76,8 +67,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     max_value_svd = 0
     max_value_svd = 0
     min_value_svd = sys.maxsize
     min_value_svd = sys.maxsize
 
 
-    image_indices = []
-
     scenes = os.listdir(path)
     scenes = os.listdir(path)
     # remove min max file from scenes folder
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
     scenes = [s for s in scenes if min_max_filename not in s]
@@ -85,23 +74,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     begin_data, end_data = p_interval
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
     begin_index, end_index = p_indices
 
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         if p_scene == folder_scene:
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             # construct each zones folder name
             zones_folder = []
             zones_folder = []
 
 
@@ -115,10 +93,14 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                 zones_folder.append(current_zone)
                 zones_folder.append(current_zone)
 
 
             images_data = []
             images_data = []
-            images_indices = []
+            images_path = []
 
 
             threshold_learned_zones = []
             threshold_learned_zones = []
 
 
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+            number_scene_image = len(scene_images)
+
             for id, zone_folder in enumerate(zones_folder):
             for id, zone_folder in enumerate(zones_folder):
 
 
                 # get threshold information
                 # get threshold information
@@ -131,27 +113,19 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned_zones.append(threshold_learned)
                     threshold_learned_zones.append(threshold_learned)
 
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_image_found = False
             threshold_image_found = False
 
 
-            file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
             svd_data = []
             svd_data = []
+           
+            # for each images
+            for id_img, img_path in enumerate(scene_images):
+                
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
 
-            while(current_counter_index <= end_counter_index):
-
-                current_counter_index_str = str(current_counter_index)
-
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+                img = Image.open(img_path)
 
 
-                image_path = file_path.format(str(current_counter_index_str))
-                img = Image.open(image_path)
-
-                svd_values = get_svd_data(p_metric, img)
+                svd_values = get_svd_data(p_feature, img)
 
 
                 if p_norm:
                 if p_norm:
                     svd_values = svd_values[begin_data:end_data]
                     svd_values = svd_values[begin_data:end_data]
@@ -167,23 +141,19 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     max_value_svd = max_value
                     max_value_svd = max_value
 
 
                 # keep in memory used data
                 # keep in memory used data
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+                        images_path.append(img_path)
                         svd_data.append(svd_values)
                         svd_data.append(svd_values)
 
 
-                    if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                    if threshold_mean < current_quality_image and not threshold_image_found:
 
 
                         threshold_image_found = True
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
+                        threshold_image_zone = dt.get_scene_image_postfix(img_path)
 
 
-                current_counter_index += step_counter
-                print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+                print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
                 sys.stdout.write("\033[F")
                 sys.stdout.write("\033[F")
 
 
-            # all indices of picture to plot
-            print(images_indices)
-
             previous_data = []
             previous_data = []
             error_data = [0.]
             error_data = [0.]
 
 
@@ -204,8 +174,7 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
 
                 # use of whole image data for computation of ssim or psnr
                 # use of whole image data for computation of ssim or psnr
                 if p_error == 'ssim' or p_error == 'psnr':
                 if p_error == 'ssim' or p_error == 'psnr':
-                    image_path = file_path.format(str(images_indices[id]))
-                    current_data = np.asarray(Image.open(image_path))
+                    current_data = np.asarray(Image.open(images_path[id]))
 
 
                 if len(previous_data) > 0:
                 if len(previous_data) > 0:
 
 
@@ -224,18 +193,21 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             ax2 = plt.subplot2grid(gridsize, (2, 0), colspan=2)
             ax2 = plt.subplot2grid(gridsize, (2, 0), colspan=2)
 
 
 
 
-            ax1.set_title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_metric + ' metric, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
+            ax1.set_title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_feature + ' feature, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
             ax1.set_ylabel('Image samples or time (minutes) generation', fontsize=14)
             ax1.set_ylabel('Image samples or time (minutes) generation', fontsize=14)
             ax1.set_xlabel('Vector features', fontsize=16)
             ax1.set_xlabel('Vector features', fontsize=16)
 
 
             for id, data in enumerate(images_data):
             for id, data in enumerate(images_data):
+                
+                current_quality_image = dt.get_scene_image_quality(images_path[id])
+                current_quality_postfix = dt.get_scene_image_postfix(images_path[id])
 
 
                 if display_error:
                 if display_error:
-                    p_label = p_scene + '_' + str(images_indices[id]) + " | " + p_error + ": " + str(error_data[id])
+                    p_label = p_scene + '_' + current_quality_postfix + " | " + p_error + ": " + str(error_data[id])
                 else:
                 else:
-                    p_label = p_scene + '_' + str(images_indices[id])
+                    p_label = p_scene + '_' + current_quality_postfix
 
 
-                if images_indices[id] == threshold_image_zone:
+                if current_quality_image == threshold_image_zone:
                     ax1.plot(data, label=p_label + " (threshold mean)", lw=4, color='red')
                     ax1.plot(data, label=p_label + " (threshold mean)", lw=4, color='red')
                 else:
                 else:
                     ax1.plot(data, label=p_label)
                     ax1.plot(data, label=p_label)
@@ -248,11 +220,11 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             ax2.set_title(p_error + " information for whole step images")
             ax2.set_title(p_error + " information for whole step images")
             ax2.set_ylabel(p_error + ' error')
             ax2.set_ylabel(p_error + ' error')
             ax2.set_xlabel('Number of samples per pixels or times')
             ax2.set_xlabel('Number of samples per pixels or times')
-            ax2.set_xticks(range(len(images_indices)))
-            ax2.set_xticklabels(list(map(int, images_indices)))
+            ax2.set_xticks(range(len(current_quality_image)))
+            ax2.set_xticklabels(list(map(dt.get_scene_image_quality, current_quality_image)))
             ax2.plot(error_data)
             ax2.plot(error_data)
 
 
-            plot_name = p_scene + '_' + p_metric + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
+            plot_name = p_scene + '_' + p_feature + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
             plt.savefig(plot_name)
             plt.savefig(plot_name)
 
 
 def main():
 def main():
@@ -262,7 +234,7 @@ def main():
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -274,14 +246,14 @@ def main():
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature   = args.feature
     p_mode     = args.mode
     p_mode     = args.mode
     p_step     = args.step
     p_step     = args.step
     p_norm     = args.norm
     p_norm     = args.norm
     p_error    = args.error
     p_error    = args.error
     p_ylim     = list(map(int, args.ylim.split(',')))
     p_ylim     = list(map(int, args.ylim.split(',')))
 
 
-    display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_error, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_error, p_ylim)
 
 
 if __name__== "__main__":
 if __name__== "__main__":
     main()
     main()

+ 39 - 68
display_svd_data_scene.py

@@ -1,34 +1,23 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
-
 import numpy as np
 import numpy as np
-import random
-import time
-import json
-import math
 
 
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing, metrics, utils
-import ipfml.iqa.fr as fr_iqa
+import matplotlib.pyplot as plt
 
 
-from skimage import color
+from data_attributes import get_svd_data
 
 
-import matplotlib as mpl
-import matplotlib.pyplot as plt
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
 
 
-from modules.utils.data import get_svd_data
-from modules.utils import config as cfg
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 
 # getting configuration information
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 min_max_filename    = cfg.min_max_filename_extension
 
 
@@ -40,19 +29,19 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 seuil_expe_filename = cfg.seuil_expe_filename
 
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 
 max_nb_bits         = 8
 max_nb_bits         = 8
 display_error       = False
 display_error       = False
 
 
 
 
-def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_ylim):
+def display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_ylim):
     """
     """
     @brief Method which gives information about svd curves from zone of picture
     @brief Method which gives information about svd curves from zone of picture
     @param p_scene, scene expected to show svd values
     @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
-    @param p_metric, metric computed to show
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_mode, normalization's mode
     @param p_norm, normalization or not of selected svd data
     @param p_norm, normalization or not of selected svd data
     @param p_ylim, ylim choice to better display of data
     @param p_ylim, ylim choice to better display of data
@@ -62,8 +51,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     max_value_svd = 0
     max_value_svd = 0
     min_value_svd = sys.maxsize
     min_value_svd = sys.maxsize
 
 
-    image_indices = []
-
     scenes = os.listdir(path)
     scenes = os.listdir(path)
     # remove min max file from scenes folder
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
     scenes = [s for s in scenes if min_max_filename not in s]
@@ -71,23 +58,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
     begin_data, end_data = p_interval
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
     begin_index, end_index = p_indices
 
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
-
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         if p_scene == folder_scene:
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             # construct each zones folder name
             zones_folder = []
             zones_folder = []
 
 
@@ -104,11 +80,14 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             images_indices = []
             images_indices = []
 
 
             threshold_learned_zones = []
             threshold_learned_zones = []
-
+    
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+            number_scene_image = len(scene_images)
+            
             for id, zone_folder in enumerate(zones_folder):
             for id, zone_folder in enumerate(zones_folder):
 
 
                 # get threshold information
                 # get threshold information
-
                 zone_path = os.path.join(scene_path, zone_folder)
                 zone_path = os.path.join(scene_path, zone_folder)
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
                 path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
 
@@ -117,27 +96,20 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned = int(seuil_file.readline().strip())
                     threshold_learned_zones.append(threshold_learned)
                     threshold_learned_zones.append(threshold_learned)
 
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_mean = np.mean(np.asarray(threshold_learned_zones))
             threshold_image_found = False
             threshold_image_found = False
 
 
-            file_path = os.path.join(scene_path, prefix_image_name + "{}.png")
-
             svd_data = []
             svd_data = []
 
 
-            while(current_counter_index <= end_counter_index):
 
 
-                current_counter_index_str = str(current_counter_index)
+            # for each images
+            for id_img, img_path in enumerate(scene_images):
+                
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
 
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+                img = Image.open(img_path)
 
 
-                image_path = file_path.format(str(current_counter_index_str))
-                img = Image.open(image_path)
-
-                svd_values = get_svd_data(p_metric, img)
+                svd_values = get_svd_data(p_feature, img)
 
 
                 if p_norm:
                 if p_norm:
                     svd_values = svd_values[begin_data:end_data]
                     svd_values = svd_values[begin_data:end_data]
@@ -155,18 +127,18 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
                     max_value_svd = max_value
                     max_value_svd = max_value
 
 
                 # keep in memory used data
                 # keep in memory used data
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+
+                        images_indices.append(dt.get_scene_image_postfix(img_path))
                         svd_data.append(svd_values)
                         svd_data.append(svd_values)
 
 
-                    if threshold_mean < int(current_counter_index) and not threshold_image_found:
+                    if threshold_mean < current_quality_image and not threshold_image_found:
 
 
                         threshold_image_found = True
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
+                        threshold_image_zone = current_quality_image
 
 
-                current_counter_index += step_counter
-                print('%.2f%%' % (current_counter_index / end_counter_index * 100))
+                print('%.2f%%' % ((id_img + 1) / number_scene_image * 100))
                 sys.stdout.write("\033[F")
                 sys.stdout.write("\033[F")
 
 
 
 
@@ -190,7 +162,6 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
 
 
 
             # display all data using matplotlib (configure plt)
             # display all data using matplotlib (configure plt)
-            #fig = plt.figure(figsize=(30, 22))
             fig, ax = plt.subplots(figsize=(30, 22))
             fig, ax = plt.subplots(figsize=(30, 22))
             ax.set_facecolor('#F9F9F9')
             ax.set_facecolor('#F9F9F9')
             #fig.patch.set_facecolor('#F9F9F9')
             #fig.patch.set_facecolor('#F9F9F9')
@@ -199,7 +170,7 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
             #plt.rc('xtick', labelsize=22)
             #plt.rc('xtick', labelsize=22)
             #plt.rc('ytick', labelsize=22)
             #plt.rc('ytick', labelsize=22)
 
 
-            #plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_metric + ' metric, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=24)
+            #plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + '], ' + p_feature + ' feature, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=24)
             ax.set_ylabel('Component values', fontsize=30)
             ax.set_ylabel('Component values', fontsize=30)
             ax.set_xlabel('Vector features', fontsize=30)
             ax.set_xlabel('Vector features', fontsize=30)
 
 
@@ -214,10 +185,10 @@ def display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step,
 
 
             plt.legend(bbox_to_anchor=(0.65, 0.98), loc=2, borderaxespad=0.2, fontsize=24)
             plt.legend(bbox_to_anchor=(0.65, 0.98), loc=2, borderaxespad=0.2, fontsize=24)
 
 
-            start_ylim, end_ylim = p_ylim
+            #start_ylim, end_ylim = p_ylim
             #ax.set_ylim(start_ylim, end_ylim)
             #ax.set_ylim(start_ylim, end_ylim)
 
 
-            plot_name = p_scene + '_' + p_metric + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
+            plot_name = p_scene + '_' + p_feature + '_' + str(p_step) + '_' + p_mode + '_' + str(p_norm) + '.png'
             plt.savefig(plot_name, facecolor=ax.get_facecolor())
             plt.savefig(plot_name, facecolor=ax.get_facecolor())
 
 
 def main():
 def main():
@@ -227,7 +198,7 @@ def main():
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--scene', type=str, help='scene index to use', choices=cfg.scenes_indices)
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -238,13 +209,13 @@ def main():
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_scene    = scenes_list[scenes_indices.index(args.scene)]
     p_indices  = list(map(int, args.indices.split(',')))
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
-    p_metric   = args.metric
+    p_feature  = args.feature
     p_mode     = args.mode
     p_mode     = args.mode
     p_step     = args.step
     p_step     = args.step
     p_norm     = args.norm
     p_norm     = args.norm
     p_ylim     = list(map(int, args.ylim.split(',')))
     p_ylim     = list(map(int, args.ylim.split(',')))
 
 
-    display_svd_values(p_scene, p_interval, p_indices, p_metric, p_mode, p_step, p_norm, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_feature, p_mode, p_step, p_norm, p_ylim)
 
 
 if __name__== "__main__":
 if __name__== "__main__":
     main()
     main()

+ 47 - 70
display_svd_zone_scene.py

@@ -1,30 +1,24 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
-
 import numpy as np
 import numpy as np
-import random
-import time
-import json
 
 
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing, metrics, utils
-from skimage import color
-
 import matplotlib.pyplot as plt
 import matplotlib.pyplot as plt
-from modules.utils.data import get_svd_data
 
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+from ipfml.processing import segmentation
+import ipfml.iqa.fr as fr_iqa
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
 
 
 # getting configuration information
 # getting configuration information
-config_filename     = cfg.config_filename
 zone_folder         = cfg.zone_folder
 zone_folder         = cfg.zone_folder
 min_max_filename    = cfg.min_max_filename_extension
 min_max_filename    = cfg.min_max_filename_extension
 
 
@@ -36,7 +30,7 @@ path                = cfg.dataset_path
 zones               = cfg.zones_indices
 zones               = cfg.zones_indices
 seuil_expe_filename = cfg.seuil_expe_filename
 seuil_expe_filename = cfg.seuil_expe_filename
 
 
-metric_choices      = cfg.metric_choices_labels
+features_choices    = cfg.features_choices_labels
 
 
 generic_output_file_svd = '_random.csv'
 generic_output_file_svd = '_random.csv'
 
 
@@ -44,7 +38,7 @@ max_nb_bits = 8
 min_value_interval = sys.maxsize
 min_value_interval = sys.maxsize
 max_value_interval = 0
 max_value_interval = 0
 
 
-def get_min_max_value_interval(_scene, _interval, _metric):
+def get_min_max_value_interval(_scene, _interval, _feature):
 
 
     global min_value_interval, max_value_interval
     global min_value_interval, max_value_interval
 
 
@@ -53,7 +47,7 @@ def get_min_max_value_interval(_scene, _interval, _metric):
     # remove min max file from scenes folder
     # remove min max file from scenes folder
     scenes = [s for s in scenes if min_max_filename not in s]
     scenes = [s for s in scenes if min_max_filename not in s]
 
 
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         # only take care of current scene
         # only take care of current scene
         if folder_scene == _scene:
         if folder_scene == _scene:
@@ -68,9 +62,9 @@ def get_min_max_value_interval(_scene, _interval, _metric):
                     index_str = "0" + index_str
                     index_str = "0" + index_str
                 zones_folder.append("zone"+index_str)
                 zones_folder.append("zone"+index_str)
 
 
-            for id_zone, zone_folder in enumerate(zones_folder):
+            for zone_folder in zones_folder:
                 zone_path = os.path.join(scene_path, zone_folder)
                 zone_path = os.path.join(scene_path, zone_folder)
-                data_filename = _metric + "_svd" + generic_output_file_svd
+                data_filename = _feature + "_svd" + generic_output_file_svd
                 data_file_path = os.path.join(zone_path, data_filename)
                 data_file_path = os.path.join(zone_path, data_filename)
 
 
                 # getting number of line and read randomly lines
                 # getting number of line and read randomly lines
@@ -83,11 +77,11 @@ def get_min_max_value_interval(_scene, _interval, _metric):
                     begin, end = _interval
                     begin, end = _interval
 
 
                     line_data = line.split(';')
                     line_data = line.split(';')
-                    metrics = line_data[begin+1:end+1]
-                    metrics = [float(m) for m in metrics]
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
 
 
-                    min_value = min(metrics)
-                    max_value = max(metrics)
+                    min_value = min(features)
+                    max_value = max(features)
 
 
                     if min_value < min_value_interval:
                     if min_value < min_value_interval:
                         min_value_interval = min_value
                         min_value_interval = min_value
@@ -96,14 +90,14 @@ def get_min_max_value_interval(_scene, _interval, _metric):
                         max_value_interval = max_value
                         max_value_interval = max_value
 
 
 
 
-def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode, p_step, p_norm, p_ylim):
+def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_feature, p_mode, p_step, p_norm, p_ylim):
     """
     """
     @brief Method which gives information about svd curves from zone of picture
     @brief Method which gives information about svd curves from zone of picture
     @param p_scene, scene expected to show svd values
     @param p_scene, scene expected to show svd values
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of svd data to display
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
     @param p_interval, interval [begin, end] of samples or minutes from render generation engine
     @param p_zone, zone's identifier of picture
     @param p_zone, zone's identifier of picture
-    @param p_metric, metric computed to show
+    @param p_feature, feature computed to show
     @param p_mode, normalization's mode
     @param p_mode, normalization's mode
     @param p_step, step of images indices
     @param p_step, step of images indices
     @param p_norm, normalization or not of selected svd data
     @param p_norm, normalization or not of selected svd data
@@ -118,23 +112,14 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
     begin_data, end_data = p_interval
     begin_data, end_data = p_interval
     begin_index, end_index = p_indices
     begin_index, end_index = p_indices
 
 
-    data_min_max_filename = os.path.join(path, p_metric + min_max_filename)
+    data_min_max_filename = os.path.join(path, p_feature + min_max_filename)
 
 
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         if p_scene == folder_scene:
         if p_scene == folder_scene:
             scene_path = os.path.join(path, folder_scene)
             scene_path = os.path.join(path, folder_scene)
 
 
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
             # construct each zones folder name
             # construct each zones folder name
             zones_folder = []
             zones_folder = []
 
 
@@ -148,15 +133,12 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
                 zones_folder.append(current_zone)
                 zones_folder.append(current_zone)
 
 
             zones_images_data = []
             zones_images_data = []
-            images_indices = []
+            images_path = []
 
 
             zone_folder = zones_folder[p_zone]
             zone_folder = zones_folder[p_zone]
 
 
             zone_path = os.path.join(scene_path, zone_folder)
             zone_path = os.path.join(scene_path, zone_folder)
 
 
-            current_counter_index = int(start_index_image)
-            end_counter_index = int(end_index_image)
-
             # get threshold information
             # get threshold information
             path_seuil = os.path.join(zone_path, seuil_expe_filename)
             path_seuil = os.path.join(zone_path, seuil_expe_filename)
 
 
@@ -166,40 +148,35 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
 
 
             threshold_image_found = False
             threshold_image_found = False
 
 
-            while(current_counter_index <= end_counter_index):
+            # get all images of folder
+            scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
 
 
-                current_counter_index_str = str(current_counter_index)
+            # for each images
+            for img_path in scene_images:
+                    
+                current_quality_image = dt.get_scene_image_quality(img_path)
 
 
-                while len(start_index_image) > len(current_counter_index_str):
-                    current_counter_index_str = "0" + current_counter_index_str
+                if current_quality_image % p_step == 0:
+                    if current_quality_image >= begin_index and current_quality_image <= end_index:
+                        images_path.append(dt.get_scene_image_postfix(img_path))
 
 
-                if current_counter_index % p_step == 0:
-                    if current_counter_index >= begin_index and current_counter_index <= end_index:
-                        images_indices.append(current_counter_index_str)
-
-                    if seuil_learned < int(current_counter_index) and not threshold_image_found:
+                    if seuil_learned < current_quality_image and not threshold_image_found:
 
 
                         threshold_image_found = True
                         threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
-
-                current_counter_index += step_counter
-
-            # all indices of picture to plot
-            print(images_indices)
+                        threshold_image_zone = dt.get_scene_image_postfix(img_path)
 
 
-            for index in images_indices:
 
 
-                img_path = os.path.join(scene_path, prefix_image_name + str(index) + ".png")
+            for img_path in images_path:
 
 
                 current_img = Image.open(img_path)
                 current_img = Image.open(img_path)
-                img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+                img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
 
                 # getting expected block id
                 # getting expected block id
                 block = img_blocks[p_zone]
                 block = img_blocks[p_zone]
 
 
                 # get data from mode
                 # get data from mode
                 # Here you can add the way you compute data
                 # Here you can add the way you compute data
-                data = get_svd_data(p_metric, block)
+                data = get_svd_data(p_feature, block)
 
 
                 # TODO : improve part of this code to get correct min / max values
                 # TODO : improve part of this code to get correct min / max values
                 if p_norm:
                 if p_norm:
@@ -230,15 +207,15 @@ def display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode,
                 else:
                 else:
                     zones_images_data.append(data)
                     zones_images_data.append(data)
 
 
-            plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + ']' + p_metric + ' metric, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
+            plt.title(p_scene + ' scene interval information SVD['+ str(begin_data) +', '+ str(end_data) +'], from scenes indices [' + str(begin_index) + ', '+ str(end_index) + ']' + p_feature + ' feature, ' + p_mode + ', with step of ' + str(p_step) + ', svd norm ' + str(p_norm), fontsize=20)
             plt.ylabel('Image samples or time (minutes) generation', fontsize=14)
             plt.ylabel('Image samples or time (minutes) generation', fontsize=14)
             plt.xlabel('Vector features', fontsize=16)
             plt.xlabel('Vector features', fontsize=16)
 
 
             for id, data in enumerate(zones_images_data):
             for id, data in enumerate(zones_images_data):
 
 
-                p_label = p_scene + "_" + images_indices[id]
+                p_label = p_scene + "_" + images_path[id]
 
 
-                if images_indices[id] == threshold_image_zone:
+                if images_path[id] == threshold_image_zone:
                     plt.plot(data, label=p_label, lw=4, color='red')
                     plt.plot(data, label=p_label, lw=4, color='red')
                 else:
                 else:
                     plt.plot(data, label=p_label)
                     plt.plot(data, label=p_label)
@@ -258,7 +235,7 @@ def main():
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
     parser.add_argument('--indices', type=str, help='Samples interval to display', default='"0, 900"')
     parser.add_argument('--zone', type=int, help='Zone to display', choices=list(range(0, 16)))
     parser.add_argument('--zone', type=int, help='Zone to display', choices=list(range(0, 16)))
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--step', type=int, help='Each step samples to display', default=10)
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
     parser.add_argument('--norm', type=int, help='If values will be normalized or not', choices=[0, 1])
@@ -270,13 +247,13 @@ def main():
     p_indices  = list(map(int, args.indices.split(',')))
     p_indices  = list(map(int, args.indices.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
     p_interval = list(map(int, args.interval.split(',')))
     p_zone     = args.zone
     p_zone     = args.zone
-    p_metric   = args.metric
+    p_feature   = args.feature
     p_mode     = args.mode
     p_mode     = args.mode
     p_step     = args.step
     p_step     = args.step
     p_norm     = args.norm
     p_norm     = args.norm
     p_ylim     = list(map(int, args.ylim.split(',')))
     p_ylim     = list(map(int, args.ylim.split(',')))
 
 
-    display_svd_values(p_scene, p_interval, p_indices, p_zone, p_metric, p_mode, p_step, p_norm, p_ylim)
+    display_svd_values(p_scene, p_interval, p_indices, p_zone, p_feature, p_mode, p_step, p_norm, p_ylim)
 
 
 if __name__== "__main__":
 if __name__== "__main__":
     main()
     main()

+ 0 - 249
display_scenes_zones.py

@@ -1,249 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
-import sys, os, argparse
-import numpy as np
-import random
-import time
-import json
-
-from PIL import Image
-from ipfml import processing, metrics, utils
-from skimage import color
-import matplotlib.pyplot as plt
-
-from modules.utils import config as cfg
-
-config_filename     = cfg.config_filename
-zone_folder         = cfg.zone_folder
-min_max_filename    = cfg.min_max_filename_extension
-
-# define all scenes values
-scenes_list         = cfg.scenes_names
-scenes_indices      = cfg.scenes_indices
-norm_choices        = cfg.normalization_choices
-path                = cfg.dataset_path
-zones               = cfg.zones_indices
-seuil_expe_filename = cfg.seuil_expe_filename
-
-metric_choices      = cfg.metric_choices_labels
-
-
-def display_data_scenes(data_type, p_scene, p_kind):
-    """
-    @brief Method which displays data from scene
-    @param data_type,  metric choice
-    @param scene, scene choice
-    @param mode, normalization choice
-    @return nothing
-    """
-
-    scenes = os.listdir(path)
-    # remove min max file from scenes folder
-    scenes = [s for s in scenes if min_max_filename not in s]
-
-    # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
-
-        if p_scene == folder_scene:
-            print(folder_scene)
-            scene_path = os.path.join(path, folder_scene)
-
-            config_file_path = os.path.join(scene_path, config_filename)
-
-            with open(config_file_path, "r") as config_file:
-                last_image_name = config_file.readline().strip()
-                prefix_image_name = config_file.readline().strip()
-                start_index_image = config_file.readline().strip()
-                end_index_image = config_file.readline().strip()
-                step_counter = int(config_file.readline().strip())
-
-            # construct each zones folder name
-            zones_folder = []
-
-            # get zones list info
-            for index in zones:
-                index_str = str(index)
-                if len(index_str) < 2:
-                    index_str = "0" + index_str
-
-                current_zone = "zone"+index_str
-                zones_folder.append(current_zone)
-
-            zones_images_data = []
-            threshold_info = []
-
-            for id_zone, zone_folder in enumerate(zones_folder):
-
-                zone_path = os.path.join(scene_path, zone_folder)
-
-                current_counter_index = int(start_index_image)
-                end_counter_index = int(end_index_image)
-
-                # get threshold information
-                path_seuil = os.path.join(zone_path, seuil_expe_filename)
-
-                # open treshold path and get this information
-                with open(path_seuil, "r") as seuil_file:
-                    seuil_learned = int(seuil_file.readline().strip())
-
-                threshold_image_found = False
-                while(current_counter_index <= end_counter_index and not threshold_image_found):
-
-                    if seuil_learned < int(current_counter_index):
-                        current_counter_index_str = str(current_counter_index)
-
-                        while len(start_index_image) > len(current_counter_index_str):
-                            current_counter_index_str = "0" + current_counter_index_str
-
-                        threshold_image_found = True
-                        threshold_image_zone = current_counter_index_str
-                        threshold_info.append(threshold_image_zone)
-
-                    current_counter_index += step_counter
-
-                # all indexes of picture to plot
-                images_indexes = [start_index_image, threshold_image_zone, end_index_image]
-                images_data = []
-
-                print(images_indexes)
-
-                for index in images_indexes:
-
-                    img_path = os.path.join(scene_path, prefix_image_name + index + ".png")
-
-                    current_img = Image.open(img_path)
-                    img_blocks = processing.divide_in_blocks(current_img, (200, 200))
-
-                    # getting expected block id
-                    block = img_blocks[id_zone]
-
-                    # get data from mode
-                    # Here you can add the way you compute data
-                    if data_type == 'lab':
-
-                        block_file_path = '/tmp/lab_img.png'
-                        block.save(block_file_path)
-                        data = processing.get_LAB_L_SVD_s(Image.open(block_file_path))
-
-                    if data_type == 'mscn_revisited':
-
-                        img_mscn_revisited = processing.rgb_to_mscn(block)
-
-                        # save tmp as img
-                        img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
-                        mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
-                        img_output.save(mscn_revisited_file_path)
-                        img_block = Image.open(mscn_revisited_file_path)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(img_block)
-
-                    '''if data_type == 'mscn':
-
-                        img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
-                        img_mscn = processing.calculate_mscn_coefficients(img_gray, 7)
-                        img_mscn_norm = utils.normalize_2D_arr(img_mscn)
-
-                        img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
-
-                        data = metrics.get_SVD_s(img_mscn_gray)'''
-
-                    if data_type == 'low_bits_6':
-
-                        low_bits_6 = processing.rgb_to_LAB_L_low_bits(block, 63)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_6)
-
-
-                    if data_type == 'low_bits_5':
-
-                        low_bits_5 = processing.rgb_to_LAB_L_low_bits(block, 31)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_5)
-
-
-                    if data_type == 'low_bits_4':
-
-                        low_bits_4 = processing.rgb_to_LAB_L_low_bits(block)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_4)
-
-                    if data_type == 'low_bits_3':
-
-                        low_bits_3 = processing.rgb_to_LAB_L_low_bits(block, 7)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_3)
-
-                    if data_type == 'low_bits_2':
-
-                        low_bits_2 = processing.rgb_to_LAB_L_low_bits(block, 3)
-
-                        # extract from temp image
-                        data = metrics.get_SVD_s(low_bits_2)
-
-                    ##################
-                    # Data mode part #
-                    ##################
-
-                    # modify data depending mode
-
-                    if p_kind == 'svdn':
-                        data = utils.normalize_arr(data)
-
-                    if p_kind == 'svdne':
-                        path_min_max = os.path.join(path, data_type + min_max_filename)
-
-                        with open(path_min_max, 'r') as f:
-                            min_val = float(f.readline())
-                            max_val = float(f.readline())
-
-                        data = utils.normalize_arr_with_range(data, min_val, max_val)
-
-                    # append of data
-                    images_data.append(data)
-
-                zones_images_data.append(images_data)
-
-            fig=plt.figure(figsize=(8, 8))
-            fig.suptitle(data_type + " values for " + p_scene + " scene (normalization : " + p_kind + ")", fontsize=20)
-
-            for id, data in enumerate(zones_images_data):
-                fig.add_subplot(4, 4, (id + 1))
-                plt.plot(data[0], label='Noisy_' + start_index_image)
-                plt.plot(data[1], label='Threshold_' + threshold_info[id])
-                plt.plot(data[2], label='Reference_' + end_index_image)
-                plt.ylabel(data_type + ' SVD, ZONE_' + str(id + 1), fontsize=18)
-                plt.xlabel('Vector features', fontsize=18)
-                plt.legend(bbox_to_anchor=(0.5, 1), loc=2, borderaxespad=0.2, fontsize=18)
-                plt.ylim(0, 0.1)
-            plt.show()
-
-def main():
-
-    parser = argparse.ArgumentParser(description="Display zones curves of metric on scene ")
-
-    parser.add_argument('--metric', type=str, help='Metric data choice', choices=metric_choices)
-    parser.add_argument('--scene', type=str, help='scene index to use', choices=scenes_indices)
-    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=norm_choices)
-
-    args = parser.parse_args()
-
-    p_metric = args.metric
-    p_kind   = args.kind
-    p_scene  = scenes_list[scenes_indices.index(args.scene)]
-
-    display_data_scenes(p_metric, p_scene, p_kind)
-
-if __name__== "__main__":
-    main()

+ 38 - 56
generate_all_data.py

@@ -1,27 +1,27 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
+# main imports
 import sys, os, argparse
 import sys, os, argparse
 import numpy as np
 import numpy as np
 import random
 import random
 import time
 import time
 import json
 import json
 
 
-from modules.utils.data import get_svd_data
+# image processing imports
 from PIL import Image
 from PIL import Image
-from ipfml import processing, metrics, utils
 from skimage import color
 from skimage import color
 
 
-from modules.utils import config as cfg
+from data_attributes import get_svd_data
+
+from ipfml.processing import transform, segmentation
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
 
 
 # getting configuration information
 # getting configuration information
-config_filename         = cfg.config_filename
 zone_folder             = cfg.zone_folder
 zone_folder             = cfg.zone_folder
 min_max_filename        = cfg.min_max_filename_extension
 min_max_filename        = cfg.min_max_filename_extension
 
 
@@ -33,7 +33,7 @@ path                    = cfg.dataset_path
 zones                   = cfg.zones_indices
 zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 seuil_expe_filename     = cfg.seuil_expe_filename
 
 
-metric_choices          = cfg.metric_choices_labels
+features_choices        = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 output_data_folder      = cfg.output_data_folder
 
 
 generic_output_file_svd = '_random.csv'
 generic_output_file_svd = '_random.csv'
@@ -41,7 +41,7 @@ generic_output_file_svd = '_random.csv'
 def generate_data_svd(data_type, mode):
 def generate_data_svd(data_type, mode):
     """
     """
     @brief Method which generates all .csv files from scenes
     @brief Method which generates all .csv files from scenes
-    @param data_type,  metric choice
+    @param data_type,  feature choice
     @param mode, normalization choice
     @param mode, normalization choice
     @return nothing
     @return nothing
     """
     """
@@ -57,20 +57,11 @@ def generate_data_svd(data_type, mode):
     data_min_max_filename = os.path.join(path, data_type + min_max_filename)
     data_min_max_filename = os.path.join(path, data_type + min_max_filename)
 
 
     # go ahead each scenes
     # go ahead each scenes
-    for id_scene, folder_scene in enumerate(scenes):
+    for folder_scene in scenes:
 
 
         print(folder_scene)
         print(folder_scene)
         scene_path = os.path.join(path, folder_scene)
         scene_path = os.path.join(path, folder_scene)
 
 
-        config_file_path = os.path.join(scene_path, config_filename)
-
-        with open(config_file_path, "r") as config_file:
-            last_image_name = config_file.readline().strip()
-            prefix_image_name = config_file.readline().strip()
-            start_index_image = config_file.readline().strip()
-            end_index_image = config_file.readline().strip()
-            step_counter = int(config_file.readline().strip())
-
         # getting output filename
         # getting output filename
         output_svd_filename = data_type + "_" + mode + generic_output_file_svd
         output_svd_filename = data_type + "_" + mode + generic_output_file_svd
 
 
@@ -93,27 +84,21 @@ def generate_data_svd(data_type, mode):
             # add writer into list
             # add writer into list
             svd_output_files.append(open(svd_file_path, 'w'))
             svd_output_files.append(open(svd_file_path, 'w'))
 
 
-
-        current_counter_index = int(start_index_image)
-        end_counter_index = int(end_index_image)
-
-
-        while(current_counter_index <= end_counter_index):
-
-            current_counter_index_str = str(current_counter_index)
-
-            while len(start_index_image) > len(current_counter_index_str):
-                current_counter_index_str = "0" + current_counter_index_str
-
-            img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+        # get all images of folder
+        scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+        number_scene_image = len(scene_images)
+            
+        for id_img, img_path in enumerate(scene_images):
+            
+            current_image_postfix = dt.get_scene_image_postfix(img_path)
 
 
             current_img = Image.open(img_path)
             current_img = Image.open(img_path)
-            img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+            img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
 
 
             for id_block, block in enumerate(img_blocks):
             for id_block, block in enumerate(img_blocks):
 
 
                 ###########################
                 ###########################
-                # Metric computation part #
+                # feature computation part #
                 ###########################
                 ###########################
 
 
                 data = get_svd_data(data_type, block)
                 data = get_svd_data(data_type, block)
@@ -151,19 +136,16 @@ def generate_data_svd(data_type, mode):
                 current_file = svd_output_files[id_block]
                 current_file = svd_output_files[id_block]
 
 
                 # add of index
                 # add of index
-                current_file.write(current_counter_index_str + ';')
+                current_file.write(current_image_postfix + ';')
 
 
                 for val in data:
                 for val in data:
                     current_file.write(str(val) + ";")
                     current_file.write(str(val) + ";")
 
 
                 current_file.write('\n')
                 current_file.write('\n')
 
 
-            start_index_image_int = int(start_index_image)
-            print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((current_counter_index - start_index_image_int) / (end_counter_index - start_index_image_int)* 100.) + "%")
+            print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((id_img + 1) / number_scene_image * 100.) + "%")
             sys.stdout.write("\033[F")
             sys.stdout.write("\033[F")
 
 
-            current_counter_index += step_counter
-
         for f in svd_output_files:
         for f in svd_output_files:
             f.close()
             f.close()
 
 
@@ -180,26 +162,26 @@ def generate_data_svd(data_type, mode):
 
 
 def main():
 def main():
 
 
-    parser = argparse.ArgumentParser(description="Compute and prepare data of metric of all scenes (keep in memory min and max value found)")
+    parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes (keep in memory min and max value found)")
 
 
-    parser.add_argument('--metric', type=str, 
-                                    help="metric choice in order to compute data (use 'all' if all metrics are needed)", 
-                                    choices=metric_choices)
+    parser.add_argument('--feature', type=str, 
+                                    help="feature choice in order to compute data (use 'all' if all features are needed)", 
+                                    choices=features_choices)
 
 
     args = parser.parse_args()
     args = parser.parse_args()
 
 
-    p_metric = args.metric
+    p_feature = args.feature
 
 
-    # generate all or specific metric data
-    if p_metric == 'all':
-        for m in metric_choices:
+    # generate all or specific feature data
+    if p_feature == 'all':
+        for m in features_choices:
             generate_data_svd(m, 'svd')
             generate_data_svd(m, 'svd')
             generate_data_svd(m, 'svdn')
             generate_data_svd(m, 'svdn')
             generate_data_svd(m, 'svdne')
             generate_data_svd(m, 'svdne')
     else:
     else:
-        generate_data_svd(p_metric, 'svd')
-        generate_data_svd(p_metric, 'svdn')
-        generate_data_svd(p_metric, 'svdne')
+        generate_data_svd(p_feature, 'svd')
+        generate_data_svd(p_feature, 'svdn')
+        generate_data_svd(p_feature, 'svdne')
 
 
 if __name__== "__main__":
 if __name__== "__main__":
     main()
     main()

generate_data_model.py → generate/generate_data_model.py


generate_data_model_corr_random.py → generate/generate_data_model_corr_random.py


generate_data_model_random.py → generate/generate_data_model_random.py


generate_data_model_random_center.py → generate/generate_data_model_random_center.py


generate_data_model_random_split.py → generate/generate_data_model_random_split.py


generate_metrics_curve.sh → generate/generate_metrics_curve.sh


+ 1 - 0
modules

@@ -0,0 +1 @@
+Subproject commit 139aa3c2312e9449b32d1d6fa506d741e7790c98

+ 0 - 0
modules/__init__.py


+ 0 - 75
modules/models.py

@@ -1,75 +0,0 @@
-from sklearn.model_selection import GridSearchCV
-from sklearn.linear_model import LogisticRegression
-from sklearn.ensemble import RandomForestClassifier, VotingClassifier
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.ensemble import GradientBoostingClassifier
-import sklearn.svm as svm
-
-
-def _get_best_model(X_train, y_train):
-
-    Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
-    gammas = [0.001, 0.01, 0.1, 1, 5, 10, 100]
-    param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
-
-    svc = svm.SVC(probability=True)
-    clf = GridSearchCV(svc, param_grid, cv=10, scoring='accuracy', verbose=10)
-
-    clf.fit(X_train, y_train)
-
-    model = clf.best_estimator_
-
-    return model
-
-def svm_model(X_train, y_train):
-
-    return _get_best_model(X_train, y_train)
-
-
-def ensemble_model(X_train, y_train):
-
-    svm_model = _get_best_model(X_train, y_train)
-
-    lr_model = LogisticRegression(solver='liblinear', multi_class='ovr', random_state=1)
-    rf_model = RandomForestClassifier(n_estimators=100, random_state=1)
-
-    ensemble_model = VotingClassifier(estimators=[
-       ('svm', svm_model), ('lr', lr_model), ('rf', rf_model)], voting='soft', weights=[1,1,1])
-
-    ensemble_model.fit(X_train, y_train)
-
-    return ensemble_model
-
-
-def ensemble_model_v2(X_train, y_train):
-
-    svm_model = _get_best_model(X_train, y_train)
-    knc_model = KNeighborsClassifier(n_neighbors=2)
-    gbc_model = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)
-    lr_model = LogisticRegression(solver='liblinear', multi_class='ovr', random_state=1)
-    rf_model = RandomForestClassifier(n_estimators=100, random_state=1)
-
-    ensemble_model = VotingClassifier(estimators=[
-       ('lr', lr_model),
-       ('knc', knc_model),
-       ('gbc', gbc_model),
-       ('svm', svm_model),
-       ('rf', rf_model)],
-       voting='soft', weights=[1, 1, 1, 1, 1])
-
-    ensemble_model.fit(X_train, y_train)
-
-    return ensemble_model
-
-def get_trained_model(choice, X_train, y_train):
-
-    if choice == 'svm_model':
-        return svm_model(X_train, y_train)
-
-    if choice == 'ensemble_model':
-        return ensemble_model(X_train, y_train)
-
-    if choice == 'ensemble_model_v2':
-        return ensemble_model_v2(X_train, y_train)
-
-

+ 0 - 0
modules/utils/__init__.py


Fichier diff supprimé car celui-ci est trop grand
+ 0 - 41
modules/utils/config.py


+ 2 - 3
predict_seuil_expe_maxwell_curve.py

@@ -124,9 +124,8 @@ def main():
                         tmp_file_path = tmp_filename.replace('__model__',  p_model_file.split('/')[-1].replace('.joblib', '_'))
                         tmp_file_path = tmp_filename.replace('__model__',  p_model_file.split('/')[-1].replace('.joblib', '_'))
                         block.save(tmp_file_path)
                         block.save(tmp_file_path)
 
 
-                        python_cmd = """python predict_noisy_image_svd.py --image {0} --interval '{1}' --model {2} --mode {3} --metric {4}""".format(tmp_file_path, p_interval, p_model_file, p_mode, p_metric)
-
-                        print(python_cmd)
+                        python_cmd_line = "python predict_noisy_image_svd.py --image {0} --interval '{1}' --model {2} --mode {3} --metric {4}"
+                        python_cmd = python_cmd_line.format(tmp_file_path, p_interval, p_model_file, p_mode, p_metric) 
 
 
                         # specify use of custom file for min max normalization
                         # specify use of custom file for min max normalization
                         if p_custom:
                         if p_custom:

+ 10 - 13
deep_network_keras_svd.py

@@ -1,3 +1,11 @@
+# main imports
+import sys, os
+import argparse
+import json
+import numpy as np
+import pandas as pd
+
+# models imports
 from keras.preprocessing.image import ImageDataGenerator
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
 from keras.models import Sequential
 from keras.layers import Conv1D, MaxPooling1D
 from keras.layers import Conv1D, MaxPooling1D
@@ -8,20 +16,9 @@ from keras import backend as K
 from sklearn.utils import shuffle
 from sklearn.utils import shuffle
 from sklearn.metrics import roc_auc_score
 from sklearn.metrics import roc_auc_score
 
 
-import numpy as np
-import pandas as pd
-
-from ipfml import processing
-import modules.utils.config as cfg
-
-from PIL import Image
-
-import sys, os
-import argparse
-import json
+# modules and config imports
+import custom_config as cfg
 
 
-import subprocess
-import time
 
 
 def f1(y_true, y_pred):
 def f1(y_true, y_pred):
     def recall(y_true, y_pred):
     def recall(y_true, y_pred):