Use of modules dependency

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "modules"]
+	path = modules
+	url = https://github.com/prise-3d/Thesis-CommonModules.git

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "/home/jbuisine/.pyenv/versions/thesis-venv/bin/python"
+}

README.md

@@ -94,6 +94,9 @@ Display threshold information about scene for each noise perceived. It's necessa
 - threshold_data_median.csv
 
 These files contains threshold information about a noise such that each row are written like that :
-- <noise>;<threshold>;<color(0, 1)>
+- noise;threshold;color(0, 1)
 
 
+## LICENSE
+
+[The MIT license](LICENSE)

analysis/.ipynb_checkpoints/noise_analysis-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data_attributes.py

@@ -0,0 +1,570 @@
+# main imports
+import numpy as np
+import sys
+
+# image transform imports
+from PIL import Image
+from skimage import color
+from sklearn.decomposition import FastICA
+from sklearn.decomposition import IncrementalPCA
+from sklearn.decomposition import TruncatedSVD
+from numpy.linalg import svd as lin_svd
+from scipy.signal import medfilt2d, wiener, cwt
+import pywt
+import cv2
+
+from ipfml.processing import transform, compression, segmentation
+from ipfml.filters import noise as nf
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
+
+def get_image_features(data_type, block):
+    """
+    Method which returns the data type expected
+    """
+
+    if data_type == 'lab':
+
+        block_file_path = '/tmp/lab_img.png'
+        block.save(block_file_path)
+        data = transform.get_LAB_L_SVD_s(Image.open(block_file_path))
+
+    if data_type == 'mscn':
+
+        img_mscn_revisited = transform.rgb_to_mscn(block)
+
+        # save tmp as img
+        img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
+        mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
+        img_output.save(mscn_revisited_file_path)
+        img_block = Image.open(mscn_revisited_file_path)
+
+        # extract from temp image
+        data = compression.get_SVD_s(img_block)
+
+    """if data_type == 'mscn':
+
+        img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
+        img_mscn = transform.calculate_mscn_coefficients(img_gray, 7)
+        img_mscn_norm = transform.normalize_2D_arr(img_mscn)
+
+        img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
+
+        data = compression.get_SVD_s(img_mscn_gray)
+    """
+
+    if data_type == 'low_bits_6':
+
+        low_bits_6 = transform.rgb_to_LAB_L_low_bits(block, 6)
+        data = compression.get_SVD_s(low_bits_6)
+
+    if data_type == 'low_bits_5':
+
+        low_bits_5 = transform.rgb_to_LAB_L_low_bits(block, 5)
+        data = compression.get_SVD_s(low_bits_5)
+
+    if data_type == 'low_bits_4':
+
+        low_bits_4 = transform.rgb_to_LAB_L_low_bits(block, 4)
+        data = compression.get_SVD_s(low_bits_4)
+
+    if data_type == 'low_bits_3':
+
+        low_bits_3 = transform.rgb_to_LAB_L_low_bits(block, 3)
+        data = compression.get_SVD_s(low_bits_3)
+
+    if data_type == 'low_bits_2':
+
+        low_bits_2 = transform.rgb_to_LAB_L_low_bits(block, 2)
+        data = compression.get_SVD_s(low_bits_2)
+
+    if data_type == 'low_bits_4_shifted_2':
+
+        data = compression.get_SVD_s(transform.rgb_to_LAB_L_bits(block, (3, 6)))
+
+    if data_type == 'sub_blocks_stats':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 4), int(height / 4)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+
+            # get information we want from svd
+            data.append(np.mean(l_svd_data))
+            data.append(np.median(l_svd_data))
+            data.append(np.percentile(l_svd_data, 25))
+            data.append(np.percentile(l_svd_data, 75))
+            data.append(np.var(l_svd_data))
+
+            area_under_curve = utils.integral_area_trapz(l_svd_data, dx=100)
+            data.append(area_under_curve)
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'sub_blocks_stats_reduced':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 4), int(height / 4)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+
+            # get information we want from svd
+            data.append(np.mean(l_svd_data))
+            data.append(np.median(l_svd_data))
+            data.append(np.percentile(l_svd_data, 25))
+            data.append(np.percentile(l_svd_data, 75))
+            data.append(np.var(l_svd_data))
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'sub_blocks_area':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 8), int(height / 8)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+
+            area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
+            data.append(area_under_curve)
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'sub_blocks_area_normed':
+
+        block = np.asarray(block)
+        width, height, _= block.shape
+        sub_width, sub_height = int(width / 8), int(height / 8)
+
+        sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height))
+
+        data = []
+
+        for sub_b in sub_blocks:
+
+            # by default use the whole lab L canal
+            l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b))
+            l_svd_data = utils.normalize_arr(l_svd_data)
+
+            area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50)
+            data.append(area_under_curve)
+
+        # convert into numpy array after computing all stats
+        data = np.asarray(data)
+
+    if data_type == 'mscn_var_4':
+
+        data = _get_mscn_variance(block, (100, 100))
+
+    if data_type == 'mscn_var_16':
+
+        data = _get_mscn_variance(block, (50, 50))
+
+    if data_type == 'mscn_var_64':
+
+        data = _get_mscn_variance(block, (25, 25))
+
+    if data_type == 'mscn_var_16_max':
+
+        data = _get_mscn_variance(block, (50, 50))
+        data = np.asarray(data)
+        size = int(len(data) / 4)
+        indices = data.argsort()[-size:][::-1]
+        data = data[indices]
+
+    if data_type == 'mscn_var_64_max':
+
+        data = _get_mscn_variance(block, (25, 25))
+        data = np.asarray(data)
+        size = int(len(data) / 4)
+        indices = data.argsort()[-size:][::-1]
+        data = data[indices]
+
+    if data_type == 'ica_diff':
+        current_image = transform.get_LAB_L(block)
+
+        ica = FastICA(n_components=50)
+        ica.fit(current_image)
+
+        image_ica = ica.fit_transform(current_image)
+        image_restored = ica.inverse_transform(image_ica)
+
+        final_image = utils.normalize_2D_arr(image_restored)
+        final_image = np.array(final_image * 255, 'uint8')
+
+        sv_values = utils.normalize_arr(compression.get_SVD_s(current_image))
+        ica_sv_values = utils.normalize_arr(compression.get_SVD_s(final_image))
+
+        data = abs(np.array(sv_values) - np.array(ica_sv_values))
+
+    if data_type == 'svd_trunc_diff':
+
+        current_image = transform.get_LAB_L(block)
+
+        svd = TruncatedSVD(n_components=30, n_iter=100, random_state=42)
+        transformed_image = svd.fit_transform(current_image)
+        restored_image = svd.inverse_transform(transformed_image)
+
+        reduced_image = (current_image - restored_image)
+
+        U, s, V = compression.get_SVD(reduced_image)
+        data = s
+
+    if data_type == 'ipca_diff':
+
+        current_image = transform.get_LAB_L(block)
+
+        transformer = IncrementalPCA(n_components=20, batch_size=25)
+        transformed_image = transformer.fit_transform(current_image)
+        restored_image = transformer.inverse_transform(transformed_image)
+
+        reduced_image = (current_image - restored_image)
+
+        U, s, V = compression.get_SVD(reduced_image)
+        data = s
+
+    if data_type == 'svd_reconstruct':
+
+        reconstructed_interval = (90, 200)
+        begin, end = reconstructed_interval
+
+        lab_img = transform.get_LAB_L(block)
+        lab_img = np.array(lab_img, 'uint8')
+
+        U, s, V = lin_svd(lab_img, full_matrices=True)
+
+        smat = np.zeros((end-begin, end-begin), dtype=complex)
+        smat[:, :] = np.diag(s[begin:end])
+        output_img = np.dot(U[:, begin:end],  np.dot(smat, V[begin:end, :]))
+
+        output_img = np.array(output_img, 'uint8')
+
+        data = compression.get_SVD_s(output_img)
+
+    if 'sv_std_filters' in data_type:
+
+        # convert into lab by default to apply filters
+        lab_img = transform.get_LAB_L(block)
+        arr = np.array(lab_img)
+        images = []
+        
+        # Apply list of filter on arr
+        images.append(medfilt2d(arr, [3, 3]))
+        images.append(medfilt2d(arr, [5, 5]))
+        images.append(wiener(arr, [3, 3]))
+        images.append(wiener(arr, [5, 5]))
+        
+        # By default computation of current block image
+        s_arr = compression.get_SVD_s(arr)
+        sv_vector = [s_arr]
+
+        # for each new image apply SVD and get SV 
+        for img in images:
+            s = compression.get_SVD_s(img)
+            sv_vector.append(s)
+            
+        sv_array = np.array(sv_vector)
+        
+        _, len = sv_array.shape
+        
+        sv_std = []
+        
+        # normalize each SV vectors and compute standard deviation for each sub vectors
+        for i in range(len):
+            sv_array[:, i] = utils.normalize_arr(sv_array[:, i])
+            sv_std.append(np.std(sv_array[:, i]))
+        
+        indices = []
+
+        if 'lowest' in data_type:
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
+
+        if 'highest' in data_type:
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
+
+        # data are arranged following std trend computed
+        data = s_arr[indices]
+
+    # with the use of wavelet
+    if 'wave_sv_std_filters' in data_type:
+
+        # convert into lab by default to apply filters
+        lab_img = transform.get_LAB_L(block)
+        arr = np.array(lab_img)
+        images = []
+        
+        # Apply list of filter on arr
+        images.append(medfilt2d(arr, [3, 3]))
+        
+        # By default computation of current block image
+        s_arr = compression.get_SVD_s(arr)
+        sv_vector = [s_arr]
+
+        # for each new image apply SVD and get SV 
+        for img in images:
+            s = compression.get_SVD_s(img)
+            sv_vector.append(s)
+            
+        sv_array = np.array(sv_vector)
+        
+        _, len = sv_array.shape
+        
+        sv_std = []
+        
+        # normalize each SV vectors and compute standard deviation for each sub vectors
+        for i in range(len):
+            sv_array[:, i] = utils.normalize_arr(sv_array[:, i])
+            sv_std.append(np.std(sv_array[:, i]))
+        
+        indices = []
+
+        if 'lowest' in data_type:
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
+
+        if 'highest' in data_type:
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
+
+        # data are arranged following std trend computed
+        data = s_arr[indices]
+
+    # with the use of wavelet
+    if 'sv_std_filters_full' in data_type:
+
+        # convert into lab by default to apply filters
+        lab_img = transform.get_LAB_L(block)
+        arr = np.array(lab_img)
+        images = []
+        
+        # Apply list of filter on arr
+        kernel = np.ones((3,3),np.float32)/9
+        images.append(cv2.filter2D(arr,-1,kernel))
+
+        kernel = np.ones((5,5),np.float32)/25
+        images.append(cv2.filter2D(arr,-1,kernel))
+
+        images.append(cv2.GaussianBlur(arr, (3, 3), 0.5))
+
+        images.append(cv2.GaussianBlur(arr, (3, 3), 1))
+
+        images.append(cv2.GaussianBlur(arr, (3, 3), 1.5))
+
+        images.append(cv2.GaussianBlur(arr, (5, 5), 0.5))
+
+        images.append(cv2.GaussianBlur(arr, (5, 5), 1))
+
+        images.append(cv2.GaussianBlur(arr, (5, 5), 1.5))
+
+        images.append(medfilt2d(arr, [3, 3]))
+
+        images.append(medfilt2d(arr, [5, 5]))
+
+        images.append(wiener(arr, [3, 3]))
+
+        images.append(wiener(arr, [5, 5]))
+
+        wave = w2d(arr, 'db1', 2)
+        images.append(np.array(wave, 'float64'))
+        
+        # By default computation of current block image
+        s_arr = compression.get_SVD_s(arr)
+        sv_vector = [s_arr]
+
+        # for each new image apply SVD and get SV 
+        for img in images:
+            s = compression.get_SVD_s(img)
+            sv_vector.append(s)
+            
+        sv_array = np.array(sv_vector)
+        
+        _, length = sv_array.shape
+        
+        sv_std = []
+        
+        # normalize each SV vectors and compute standard deviation for each sub vectors
+        for i in range(length):
+            sv_array[:, i] = utils.normalize_arr(sv_array[:, i])
+            sv_std.append(np.std(sv_array[:, i]))
+        
+        indices = []
+
+        if 'lowest' in data_type:
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
+
+        if 'highest' in data_type:
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
+
+        # data are arranged following std trend computed
+        data = s_arr[indices]
+
+    if 'sv_entropy_std_filters' in data_type:
+
+        lab_img = transform.get_LAB_L(block)
+        arr = np.array(lab_img)
+
+        images = []
+
+        kernel = np.ones((3,3),np.float32)/9
+        images.append(cv2.filter2D(arr,-1,kernel))
+
+        kernel = np.ones((5,5),np.float32)/25
+        images.append(cv2.filter2D(arr,-1,kernel))
+
+        images.append(cv2.GaussianBlur(arr, (3, 3), 0.5))
+
+        images.append(cv2.GaussianBlur(arr, (3, 3), 1))
+
+        images.append(cv2.GaussianBlur(arr, (3, 3), 1.5))
+
+        images.append(cv2.GaussianBlur(arr, (5, 5), 0.5))
+
+        images.append(cv2.GaussianBlur(arr, (5, 5), 1))
+
+        images.append(cv2.GaussianBlur(arr, (5, 5), 1.5))
+
+        images.append(medfilt2d(arr, [3, 3]))
+
+        images.append(medfilt2d(arr, [5, 5]))
+
+        images.append(wiener(arr, [3, 3]))
+
+        images.append(wiener(arr, [5, 5]))
+
+        wave = w2d(arr, 'db1', 2)
+        images.append(np.array(wave, 'float64'))
+
+        sv_vector = []
+        sv_entropy_list = []
+        
+        # for each new image apply SVD and get SV 
+        for img in images:
+            s = compression.get_SVD_s(img)
+            sv_vector.append(s)
+
+            sv_entropy = [utils.get_entropy_contribution_of_i(s, id_sv) for id_sv, sv in enumerate(s)]
+            sv_entropy_list.append(sv_entropy)
+        
+        sv_std = []
+        
+        sv_array = np.array(sv_vector)
+        _, length = sv_array.shape
+        
+        # normalize each SV vectors and compute standard deviation for each sub vectors
+        for i in range(length):
+            sv_array[:, i] = utils.normalize_arr(sv_array[:, i])
+            sv_std.append(np.std(sv_array[:, i]))
+        
+        indices = []
+
+        if 'lowest' in data_type:
+            indices = utils.get_indices_of_lowest_values(sv_std, 200)
+
+        if 'highest' in data_type:
+            indices = utils.get_indices_of_highest_values(sv_std, 200)
+
+        # data are arranged following std trend computed
+        s_arr = compression.get_SVD_s(arr)
+        data = s_arr[indices]
+
+    return data
+
+
+def get_noise_result(_image, _n, _noise_choice, _identical=False, _p=None):
+    """Return image with applied noise using choice
+
+    Args:
+        image: image to convert
+        _n: importance of noise expected [1, 999]
+        _noise_choice: choise of noise filter to apply
+        _identical: specify if the distribution is the same or not for each chanel
+        _p: optional parameter for salt_pepper noise
+
+    Returns:
+        Noisy image with noise filter applied
+
+    """
+
+    noise_method = None
+    function_name = _noise_choice + '_noise'
+
+    try:
+        noise_method = getattr(nf, function_name)
+    except AttributeError:
+        raise NotImplementedError("Noise filter `{}` not implement `{}`".format(nf.__name__, function_name))
+
+
+    if _p:
+
+        if _noise_choice != 'salt_pepper':
+            raise ValueError("p parameter is only used for salt pepper noise...")
+
+        return noise_method(_image, _n, identical=_identical, p=_p)
+    else:
+        return noise_method(_image, _n, identical=_identical)
+
+
+# others functions
+
+def w2d(arr, mode='haar', level=1):
+    #convert to float   
+    imArray = arr
+    np.divide(imArray, 255)
+
+    # compute coefficients 
+    coeffs=pywt.wavedec2(imArray, mode, level=level)
+
+    #Process Coefficients
+    coeffs_H=list(coeffs)  
+    coeffs_H[0] *= 0
+
+    # reconstruction
+    imArray_H = pywt.waverec2(coeffs_H, mode)
+    imArray_H *= 255
+    imArray_H = np.uint8(imArray_H)
+
+    return imArray_H
+
+def _get_mscn_variance(block, sub_block_size=(50, 50)):
+
+    blocks = segmentation.divide_in_blocks(block, sub_block_size)
+
+    data = []
+
+    for block in blocks:
+        mscn_coefficients = transform.get_mscn_coefficients(block)
+        flat_coeff = mscn_coefficients.flatten()
+        data.append(np.var(flat_coeff))
+
+    return np.sort(data)
+

display/noise_svd_tend_visualization.py

@@ -0,0 +1,201 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+
+# image processing imports
+from PIL import Image
+import matplotlib.pyplot as plt
+
+from ipfml import utils
+import ipfml.iqa.fr as fr_iqa
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from data_attributes import get_image_features
+
+
+# others variables
+noise_list            = cfg.noise_labels
+generated_folder      = cfg.generated_folder
+filename_ext          = cfg.filename_ext
+feature_choices       = cfg.features_choices_labels
+normalization_choices = cfg.normalization_choices
+pictures_folder       = cfg.pictures_output_folder
+error_data_choices    = cfg.error_data_choices
+
+steparam_picture          = 10
+
+
+
+def get_error_distance(param_error, y_true, y_test):
+
+    function_name = param_error
+
+    try:
+        error_method = getattr(fr_iqa, function_name)
+    except AttributeError:
+        raise NotImplementedError("Error method `{}` not implement `{}`".format(fr_iqa.__name__, function_name))
+
+    return error_method(y_true, y_test)
+
+
+def main():
+
+    max_value_svd = 0
+    min_value_svd = sys.maxsize
+
+    parser = argparse.ArgumentParser(description="Display svd tend of images with noise level")
+
+    parser.add_argument('--prefix', type=str, help='Generated noise folder prefix (ex: `generated/prefix/noise`)')
+    parser.add_argument('--mode', type=str, help='Kind of normalization', default=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature choice', default=feature_choices)
+    parser.add_argument('--n', type=int, help='Number of images')
+    parser.add_argument('--color', type=int, help='Use of color or grey level', default=0)
+    parser.add_argument('--norm', type=int, help='Use of normalization from interval or whole data vector', default=0)
+    parser.add_argument('--interval', type=str, help='Interval data choice (ex: `0, 200`)', default="0, 200")
+    parser.add_argument('--step', type=int, help='Step of image indices to keep', default=1)
+    parser.add_argument('--ylim', type=str, help='Limite to display data (ex: `0, 1`)', default="0, 1")
+    parser.add_argument('--error', type=str, help='Error used for information data', default=error_data_choices)
+
+    args = parser.parse_args()
+
+    param_prefix   = args.prefix
+    param_mode     = args.mode
+    param_feature  = args.feature
+    param_n        = args.n
+    param_color    = args.color
+    param_norm     = args.norm
+    param_interval = list(map(int, args.interval.split(',')))
+    param_step     = args.step
+    param_ylim     = list(map(float, args.ylim.split(',')))
+    param_error    = args.error
+
+
+    param_prefix = param_prefix.split('/')[1].replace('_', '')
+    noise_name = param_prefix.split('/')[2]
+
+    if param_color:
+        file_path = os.path.join(param_prefix, param_prefix + "_" + noise_name + "_color_{}." + filename_ext)
+    else:
+        file_path = os.path.join(param_prefix, param_prefix + "_" + noise_name + "_{}." + filename_ext)
+
+    begin, end = param_interval
+    all_svd_data = []
+
+    svd_data = []
+    image_indices = []
+
+    noise_indices = range(1, param_n)[::-1]
+
+    # get all data from images
+    for i in noise_indices:
+
+        if i % steparam_picture == 0:
+
+            image_path = file_path.format(str(i))
+
+            img = Image.open(image_path)
+
+            svd_values = get_image_features(param_feature, img)
+
+            if param_norm:
+                svd_values = svd_values[begin:end]
+
+            all_svd_data.append(svd_values)
+
+            # update min max values
+            min_value = svd_values.min()
+            max_value = svd_values.max()
+
+            if min_value < min_value_svd:
+                min_value_svd = min_value
+
+            if max_value > max_value_svd:
+                max_value_svd = max_value
+
+        print('%.2f%%' % ((param_n - i + 1) / param_n * 100))
+        sys.stdout.write("\033[F")
+
+    previous_data = []
+    error_data = [0.]
+
+    for id, data in enumerate(all_svd_data):
+
+        current_id = (param_n - ((id + 1) * 10))
+
+        if current_id % param_step == 0:
+
+            current_data = data
+
+            if param_mode == 'svdn':
+                current_data = utils.normalize_arr(current_data)
+
+            if param_mode == 'svdne':
+                current_data = utils.normalize_arr_with_range(current_data, min_value_svd, max_value_svd)
+
+            svd_data.append(current_data)
+            image_indices.append(current_id)
+
+            # use of whole image data for computation of ssim or psnr
+            if param_error == 'ssim' or param_error == 'psnr':
+                image_path = file_path.format(str(current_id))
+                current_data = np.asarray(Image.open(image_path))
+
+            if len(previous_data) > 0:
+
+                current_error = get_error_distance(param_error, previous_data, current_data)
+                error_data.append(current_error)
+
+            if len(previous_data) == 0:
+                previous_data = current_data
+
+    # display all data using matplotlib (configure plt)
+    gridsize = (3, 2)
+
+    # fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(30, 22))
+    fig = plt.figure(figsize=(30, 22))
+    ax1 = plt.subplot2grid(gridsize, (0, 0), colspan=2, rowspan=2)
+    ax2 = plt.subplot2grid(gridsize, (2, 0), colspan=2)
+
+    ax1.set_title(param_prefix  + ', ' + noise_name + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + param_feature + ' feature, step ' + str(param_step) + ' normalization ' + param_mode)
+    ax1.set_label('Importance of noise [1, 999]')
+    ax1.set_xlabel('Vector features')
+
+    for id, data in enumerate(svd_data):
+
+        param_label = param_prefix + str(image_indices[id]) + " | " + param_error + ": " + str(error_data[id])
+        ax1.plot(data, label=param_label)
+
+    ax1.legend(bbox_to_anchor=(0.75, 1), loc=2, borderaxespad=0.2, fontsize=12)
+
+    if not param_norm:
+        ax1.set_xlim(begin, end)
+
+    # adapt ylim
+    y_begin, y_end = param_ylim
+    ax1.set_ylim(y_begin, y_end)
+
+    output_filename = param_prefix + "_" + noise_name + "_1_to_" + str(param_n) + "_B" + str(begin) + "_E" + str(end) + "_" + param_feature + "_S" + str(param_step) + "_norm" + str(param_norm )+  "_" + param_mode + "_" + param_error
+
+    if param_color:
+        output_filename = output_filename + '_color'
+
+    ax2.set_title(param_error + " information for : " + param_prefix  + ', ' + noise_name + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + param_feature + ' feature, step ' + str(param_step) + ', normalization ' + param_mode)
+    ax2.set_ylabel(param_error + ' error')
+    ax2.set_xlabel('Number of samples per pixels')
+    ax2.set_xticks(range(len(image_indices)))
+    ax2.set_xticklabels(image_indices)
+    ax2.plot(error_data)
+
+    print("Generation of output figure... %s" % output_filename)
+    output_path = os.path.join(pictures_folder, output_filename)
+
+    if not os.path.exists(pictures_folder):
+        os.makedirs(pictures_folder)
+
+    fig.savefig(output_path, dpi=(200))
+
+if __name__== "__main__":
+    main()

noise_svd_threshold.py

@@ -1,22 +1,27 @@
-import sys, os, getopt
-from PIL import Image
+# main imports
+import sys, os, argparse
+import numpy as np
 
+# image processing imports
+from PIL import Image
 from ipfml import processing, utils
+import matplotlib.pyplot as plt
 
-from modules.utils import config as cfg
-from modules.utils import data_type as dt
-from modules import noise
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from data_attributes import get_image_features
 
-import matplotlib.pyplot as plt
 
 noise_list            = cfg.noise_labels
 generated_folder      = cfg.generated_folder
 filename_ext          = cfg.filename_ext
-metric_choices        = cfg.metric_choices_labels
+feature_choices       = cfg.features_choices_labels
 normalization_choices = cfg.normalization_choices
 pictures_folder       = cfg.pictures_output_folder
 
-step_picture          = 10
+steparam_picture          = 10
 
 class ThresholdData():
     """
@@ -40,65 +45,40 @@ class ThresholdData():
 
 def main():
 
-    # default values
-    p_step = 1
-    p_color = 0
-    p_norm = 0
-    p_ylim = (0, 1)
-    p_n = 1000
-
-    if len(sys.argv) <= 1:
-        print('python noise_svd_threshold.py --prefix generated/scene --file threshold_file --metric lab --mode svdn --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1"')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "h:p:f:m:m:i:s:c:n:y", ["help=", "prefix=", "file=", "metric=", "mode=", "interval=", "step=", "color=", "norm=", "ylim="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python noise_svd_threshold.py --prefix generated/scene --file threshold_file --metric lab --mode svdn --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1"')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python noise_svd_threshold.py --prefix generated/scene --file threshold_file --metric lab --mode svdn --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1"')
-            sys.exit()
-        elif o in ("-p", "--prefix"):
-            p_path = a
-        elif o in ("-f", "--file"):
-            p_data_file = a
-
-        elif o in ("-m", "--mode"):
-            p_mode = a
-
-            if not p_mode in normalization_choices:
-                assert False, "Unknown normalization choice, %s" % normalization_choices
-
-        elif o in ("-m", "--metric"):
-            p_metric = a
-
-            if not p_metric in metric_choices:
-                assert False, "Unknown metric choice, %s" % metric_choices
-
-        elif o in ("-n", "--norm"):
-            p_norm = int(a)
-        elif o in ("-c", "--color"):
-            p_color = int(a)
-        elif o in ("-i", "--interval"):
-            p_interval = list(map(int, a.split(',')))
-        elif o in ("-s", "--step"):
-            p_step = int(a)
-        elif o in ("-y", "--ylim"):
-            p_ylim = list(map(float, a.split(',')))
-        else:
-            assert False, "unhandled option"
-
-
-    p_prefix = p_path.split('/')[1].replace('_', '')
-
-    if p_color:
-        file_path = p_path + "{}/" + p_prefix + "_{}_color_{}." + filename_ext
+    parser = argparse.ArgumentParser(description="Display threshold svd data")
+
+    parser.add_argument('--prefix', type=str, help='Generated noise folder prefix (ex: `generated/prefix/noise`)')
+    parser.add_argument('--file', type=str, help='Threshold file to use')
+    parser.add_argument('--mode', type=str, help='Kind of normalization', default=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature choice', default=feature_choices)
+    parser.add_argument('--color', type=int, help='Use of color or grey level', default=0)
+    parser.add_argument('--norm', type=int, help='Use of normalization from interval or whole data vector', default=0)
+    parser.add_argument('--interval', type=str, help='Interval data choice (ex: `0, 200`)', default="0, 200")
+    parser.add_argument('--step', type=int, help='Step of image indices to keep', default=1)
+    parser.add_argument('--ylim', type=str, help='Limite to display data (ex: `0, 1`)', default="0, 1")
+
+    args = parser.parse_args()
+
+    param_prefix   = args.prefix
+    param_file     = args.file
+    param_mode     = args.mode
+    param_feature  = args.feature
+    param_n        = args.n
+    param_color    = args.color
+    param_norm     = args.norm
+    param_interval = list(map(int, args.interval.split(',')))
+    param_step     = args.step
+    param_ylim     = list(map(float, args.ylim.split(',')))
+
+
+    param_prefix = param_prefix.split('/')[1].replace('_', '')
+
+    if param_color:
+        file_path = param_prefix + "{}/" + param_prefix + "_{}_color_{}." + filename_ext
     else:
-        file_path = p_path + "{}/" + p_prefix + "_{}_{}." + filename_ext
+        file_path = param_prefix + "{}/" + param_prefix + "_{}_{}." + filename_ext
 
-    begin, end = p_interval
+    begin, end = param_interval
 
     svd_data = []
     final_svd_data = []
@@ -108,7 +88,7 @@ def main():
     threshold_data = []
 
     # read data threshold file
-    with open(p_data_file, 'r') as f:
+    with open(param_file, 'r') as f:
         lines = f.readlines()
 
         for line in lines:
@@ -119,7 +99,7 @@ def main():
             threshold_data.append(threshold)
 
     # filter data if color or not
-    threshold_data = [t for t in threshold_data if t.isColor() == p_color]
+    threshold_data = [t for t in threshold_data if t.isColor() == param_color]
 
     for id, threshold in enumerate(threshold_data):
 
@@ -130,15 +110,15 @@ def main():
         threshold_found = False
 
         # get all data from images
-        for i in range(1, p_n):
+        for i in range(1, param_n):
 
-            if i % step_picture == 0:
+            if i % steparam_picture == 0:
                 image_path = file_path.format(current_noise, current_noise, str(i))
                 img = Image.open(image_path)
 
-                svd_values = dt.get_svd_data(p_metric, img)
+                svd_values = get_image_features(param_feature, img)
 
-                if p_norm:
+                if param_norm:
                     svd_values = svd_values[begin:end]
 
                 # only append data once
@@ -164,7 +144,7 @@ def main():
 
                 min_max_list[current_noise] = (current_min, current_max)
 
-            print('%.2f%%' % (((i + 1) * 100 + (id * p_n * 100)) / (p_n * len(threshold_data))))
+            print('%.2f%%' % (((i + 1) * 100 + (id * param_n * 100)) / (param_n * len(threshold_data))))
             sys.stdout.write("\033[F")
 
     for id, data in enumerate(svd_data):
@@ -174,10 +154,10 @@ def main():
         threshold = threshold_data[id]
         min_value_svd, max_value_svd = min_max_list[threshold.get_noise()]
 
-        if p_mode == 'svdn':
+        if param_mode == 'svdn':
             current_data = utils.normalize_arr(current_data)
 
-        if p_mode == 'svdne':
+        if param_mode == 'svdne':
             current_data = utils.normalize_arr_with_range(current_data, min_value_svd, max_value_svd)
 
         final_svd_data.append(current_data)
@@ -186,27 +166,27 @@ def main():
 
     plt.rcParams['figure.figsize'] = (25, 18)
 
-    plt.title(p_prefix  + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + p_metric + ' metric, step ' + str(p_step) + ' normalization ' + p_mode, fontsize=20)
+    plt.title(param_prefix  + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + param_feature + ' feature, step ' + str(param_step) + ' normalization ' + param_mode, fontsize=20)
     plt.ylabel('Importance of noise [1, 999]', fontsize=14)
     plt.xlabel('Vector features', fontsize=16)
 
     for id, data in enumerate(final_svd_data):
 
-        p_label = p_prefix + '_' + threshold_data[id].get_noise() + str(image_indices[id])
-        plt.plot(data, label=p_label)
+        param_label = param_prefix + '_' + threshold_data[id].get_noise() + str(image_indices[id])
+        plt.plot(data, label=param_label)
 
     plt.legend(bbox_to_anchor=(0.8, 1), loc=2, borderaxespad=0.2, fontsize=14)
 
-    if not p_norm:
+    if not param_norm:
         plt.xlim(begin, end)
 
     # adapt ylim
-    y_begin, y_end = p_ylim
+    y_begin, y_end = param_ylim
     plt.ylim(y_begin, y_end)
 
-    output_filename = p_prefix + "_threshold_1_to_" + str(p_n) + "_B" + str(begin) + "_E" + str(end) + "_" + p_metric + "_S" + str(p_step) + "_norm" + str(p_norm )+  "_" + p_mode
+    output_filename = param_prefix + "_threshold_1_to_" + str(param_n) + "_B" + str(begin) + "_E" + str(end) + "_" + param_feature + "_S" + str(param_step) + "_norm" + str(param_norm )+  "_" + param_mode
 
-    if p_color:
+    if param_color:
         output_filename = output_filename + '_color'
 
     print("Generation of output figure... %s" % output_filename)

display/noise_svd_visualization.py

@@ -0,0 +1,146 @@
+# main imports
+import sys, os, argparse
+
+# image processing imports
+from PIL import Image
+import matplotlib.pyplot as plt
+
+from ipfml import processing, utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from data_attributes import get_image_features
+
+# other variables
+noise_list            = cfg.noise_labels
+generated_folder      = cfg.generated_folder
+filename_ext          = cfg.filename_ext
+feature_choices       = cfg.features_choices_labels
+normalization_choices = cfg.normalization_choices
+pictures_folder       = cfg.pictures_output_folder
+
+steparam_picture          = 10
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Display svd of images with noise level")
+
+    parser.add_argument('--prefix', type=str, help='Generated noise folder prefix (ex: `generated/prefix/noise`)')
+    parser.add_argument('--mode', type=str, help='Kind of normalization', default=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature choice', default=feature_choices)
+    parser.add_argument('--n', type=int, help='Number of images')
+    parser.add_argument('--color', type=int, help='Use of color or grey level', default=0)
+    parser.add_argument('--norm', type=int, help='Use of normalization from interval or whole data vector', default=0)
+    parser.add_argument('--interval', type=str, help='Interval data choice (ex: `0, 200`)', default="0, 200")
+    parser.add_argument('--step', type=int, help='Step of image indices to keep', default=1)
+    parser.add_argument('--ylim', type=str, help='Limite to display data (ex: `0, 1`)', default="0, 1")
+
+    args = parser.parse_args()
+
+    param_prefix   = args.prefix
+    param_mode     = args.mode
+    param_feature  = args.feature
+    param_n        = args.n
+    param_color    = args.color
+    param_norm     = args.norm
+    param_interval = list(map(int, args.interval.split(',')))
+    param_step     = args.step
+    param_ylim     = list(map(float, args.ylim.split(',')))
+
+
+    param_prefix = param_prefix.split('/')[1].replace('_', '')
+    noise_name = param_prefix.split('/')[2]
+
+    if param_color:
+        file_path = param_prefix + "/" + param_prefix + "_" + noise_name + "_color_{}." + filename_ext
+    else:
+        file_path = param_prefix + "/" + param_prefix + "_" + noise_name + "_{}." + filename_ext
+
+    begin, end = param_interval
+    all_svd_data = []
+
+    svd_data = []
+    image_indices = []
+
+    # get all data from images
+    for i in range(1, param_n):
+
+        if i % steparam_picture == 0:
+
+            image_path = file_path.format(str(i))
+            img = Image.open(image_path)
+
+            svd_values = get_image_features(param_feature, img)
+
+            if param_norm:
+                svd_values = svd_values[begin:end]
+
+            all_svd_data.append(svd_values)
+
+            # update min max values
+            min_value = svd_values.min()
+            max_value = svd_values.max()
+
+            if min_value < min_value_svd:
+                min_value_svd = min_value
+
+            if max_value > max_value_svd:
+                max_value_svd = max_value
+
+            print('%.2f%%' % ((i + 1) / param_n * 100))
+            sys.stdout.write("\033[F")
+
+    for id, data in enumerate(all_svd_data):
+
+        if (id * steparam_picture) % param_step == 0:
+
+            current_data = data
+            if param_mode == 'svdn':
+                current_data = utils.normalize_arr(current_data)
+
+            if param_mode == 'svdne':
+                current_data = utils.normalize_arr_with_range(current_data, min_value_svd, max_value_svd)
+
+            svd_data.append(current_data)
+            image_indices.append(str(id * steparam_picture))
+
+    # display all data using matplotlib (configure plt)
+
+    plt.rcParams['figure.figsize'] = (25, 18)
+
+    plt.title(param_prefix  + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + param_feature + ' feature, step ' + str(param_step) + ' normalization ' + param_mode, fontsize=20)
+    plt.ylabel('Importance of noise [1, 999]', fontsize=14)
+    plt.xlabel('Vector features', fontsize=16)
+
+    for id, data in enumerate(svd_data):
+
+        param_label = param_prefix + str(image_indices[id])
+        plt.plot(data, label=param_label)
+
+    plt.legend(bbox_to_anchor=(0.8, 1), loc=2, borderaxespad=0.2, fontsize=14)
+
+    if not param_norm:
+        plt.xlim(begin, end)
+
+    # adapt ylim
+    y_begin, y_end = param_ylim
+    plt.ylim(y_begin, y_end)
+
+    output_filename = param_prefix + "_" + noise_name + "_1_to_" + str(param_n) + "_B" + str(begin) + "_E" + str(end) + "_" + param_feature + "_S" + str(param_step) + "_norm" + str(param_norm )+  "_" + param_mode
+
+    if param_color:
+        output_filename = output_filename + '_color'
+
+    print("Generation of output figure... %s" % output_filename)
+    output_path = os.path.join(pictures_folder, output_filename)
+
+    if not os.path.exists(pictures_folder):
+        os.makedirs(pictures_folder)
+
+    plt.savefig(output_path, dpi=(200))
+
+
+if __name__== "__main__":
+    main()

generate_all_data.py

@@ -1,24 +1,22 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
-import sys, os, getopt
+# main imports
+import sys, os, argparse
 import numpy as np
 import random
 import time
 import json
 
-from modules.utils.data_type import get_svd_data
+# image processing imports
 from PIL import Image
-from ipfml import processing, metrics, utils
 from skimage import color
 
-from modules.utils import config as cfg
+from ipfml.processing.segmentation import divide_in_blocks
+from ipfml import utils
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from data_attributes import get_image_features
 
 # getting configuration information
 zone_folder             = cfg.zone_folder
@@ -33,7 +31,7 @@ zones                   = cfg.zones_indices
 seuil_expe_filename     = cfg.seuil_expe_filename
 
 noise_choices           = cfg.noise_labels
-metric_choices          = cfg.metric_choices_labels
+feature_choices         = cfg.features_choices_labels
 output_data_folder      = cfg.output_data_folder
 
 end_counter_index       = cfg.default_number_of_images
@@ -47,7 +45,7 @@ calibration_folder      = 'calibration'
 def generate_data_svd(data_type, color, mode):
     """
     @brief Method which generates all .csv files from scenes
-    @param data_type,  metric choice
+    @param data_type,  feature choice
     @param mode, normalization choice
     @return nothing
     """
@@ -118,15 +116,15 @@ def generate_data_svd(data_type, color, mode):
                         img_path = os.path.join(noise_path, folder_scene + "_" + noise + "_" + counter_index_str + ".png")
 
                     current_img = Image.open(img_path)
-                    img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+                    img_blocks = divide_in_blocks(current_img, (200, 200))
 
                     for id_block, block in enumerate(img_blocks):
 
                         ###########################
-                        # Metric computation part #
+                        # feature computation part #
                         ###########################
 
-                        data = get_svd_data(data_type, block)
+                        data = get_image_features(data_type, block)
 
                         ##################
                         # Data mode part #
@@ -197,54 +195,36 @@ def generate_data_svd(data_type, color, mode):
 
 def main():
 
-    # default value of p_step
-    p_step = 10
-    p_color = 0
-
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('python generate_all_data.py --metric all --color 0')
-        print('python generate_all_data.py --metric lab --color 0')
-        print('python generate_all_data.py --metric lab --color 1 --step 10')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "hm:s:c", ["help=", "metric=", "step=", "color="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python generate_all_data.py --metric all --color 1 --step 10')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python generate_all_data.py --metric all --color 1 --step 10')
-            sys.exit()
-        elif o in ("-s", "--step"):
-            p_step = int(a)
-        elif o in ("-c", "--color"):
-            p_color = int(a)
-        elif o in ("-m", "--metric"):
-            p_metric = a
-
-            if p_metric != 'all' and p_metric not in metric_choices:
-                assert False, "Invalid metric choice"
-        else:
-            assert False, "unhandled option"
+    parser = argparse.ArgumentParser(description="Compute feature on images dataset")
+
+    parser.add_argument('--feature', type=str, help='Feature choice (`all` if all features wished)')
+    parser.add_argument('--color', type=int, help='Specify if image use color or not', default=0)
+    parser.add_argument('--step', type=int, help='Step of image indices to keep', default=10)
+    args = parser.parse_args()
+
+    param_feature = args.feature
+    param_color   = args.color
+    param_step    = args.step
 
+    if param_feature != 'all' and param_feature not in feature_choices:
+        raise ValueError("Invalid feature choice ", feature_choices)
+        
     global picture_step
-    picture_step = p_step
+    picture_step = param_step
 
     if picture_step % 10 != 0:
-        assert False, "Picture step variable needs to be divided by ten"
-
-    # generate all or specific metric data
-    if p_metric == 'all':
-        for m in metric_choices:
-            generate_data_svd(m, p_color, 'svd')
-            generate_data_svd(m, p_color, 'svdn')
-            generate_data_svd(m, p_color, 'svdne')
+        raise ValueError("Picture step variable needs to be divided by ten")
+
+    # generate all or specific feature data
+    if param_feature == 'all':
+        for m in feature_choices:
+            generate_data_svd(m, param_color, 'svd')
+            generate_data_svd(m, param_color, 'svdn')
+            generate_data_svd(m, param_color, 'svdne')
     else:
-        generate_data_svd(p_metric, p_color, 'svd')
-        generate_data_svd(p_metric, p_color, 'svdn')
-        generate_data_svd(p_metric, p_color, 'svdne')
+        generate_data_svd(param_feature, param_color, 'svd')
+        generate_data_svd(param_feature, param_color, 'svdn')
+        generate_data_svd(param_feature, param_color, 'svdne')
 
 if __name__== "__main__":
     main()

generate/generate_data_model.py

@@ -0,0 +1,272 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+import pandas as pd
+import random
+
+# image processing imports
+from PIL import Image
+
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+from data_attributes import get_image_features
+
+
+# getting configuration information
+learned_folder          = cfg.learned_zones_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes variables
+scenes_list             = cfg.scenes_names
+scenes_indexes          = cfg.scenes_indices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+renderer_choices        = cfg.renderer_choices
+normalization_choices   = cfg.normalization_choices
+features_choices        = cfg.features_choices_labels
+output_data_folder      = cfg.output_data_folder
+custom_min_max_folder   = cfg.min_max_custom_folder
+min_max_ext             = cfg.min_max_filename_extension
+zones_indices           = cfg.zones_indices
+
+generic_output_file_svd = '_random.csv'
+
+min_value_interval = sys.maxsize
+max_value_interval = 0
+
+def construct_new_line(path_seuil, interval, line, choice, each, norm):
+    begin, end = interval
+
+    line_data = line.split(';')
+    seuil = line_data[0]
+    features = line_data[begin+1:end+1]
+
+    features = [float(m) for id, m in enumerate(features) if id % each == 0 ]
+
+    if norm:
+        if choice == 'svdne':
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
+        if choice == 'svdn':
+            features = utils.normalize_arr(features)
+
+    with open(path_seuil, "r") as seuil_file:
+        seuil_learned = int(seuil_file.readline().strip())
+
+    if seuil_learned > int(seuil):
+        line = '1'
+    else:
+        line = '0'
+
+    for val in features:
+        line += ';'
+        line += str(val)
+    line += '\n'
+
+    return line
+
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
+
+    global min_value_interval, max_value_interval
+
+    scenes = os.listdir(path)
+
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    for folder_scene in scenes:
+
+        # only take care of maxwell scenes
+        if folder_scene in _scenes_list:
+
+            scene_path = os.path.join(path, folder_scene)
+
+            zones_folder = []
+            # create zones list
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+                zones_folder.append("zone"+index_str)
+
+            for zone_folder in zones_folder:
+                zone_path = os.path.join(scene_path, zone_folder)
+                data_filename = _feature + "_svd" + generic_output_file_svd
+                data_file_path = os.path.join(zone_path, data_filename)
+
+                # getting number of line and read randomly lines
+                f = open(data_file_path)
+                lines = f.readlines()
+
+                # check if user select current scene and zone to be part of training data set
+                for line in lines:
+
+                    begin, end = _interval
+
+                    line_data = line.split(';')
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
+
+                    min_value = min(features)
+                    max_value = max(features)
+
+                    if min_value < min_value_interval:
+                        min_value_interval = min_value
+
+                    if max_value > max_value_interval:
+                        max_value_interval = max_value
+
+
+def generate_data_model(_filename, _interval, _choice, _feature, _scenes = scenes_list, _zones = zones_indices, _percent = 1, _step=1, _each=1, _norm=False, _custom=False):
+
+    output_train_filename = _filename + ".train"
+    output_test_filename = _filename + ".test"
+
+    if not '/' in output_train_filename:
+        raise Exception("Please select filename with directory path to save data. Example : data/dataset")
+
+    # create path if not exists
+    if not os.path.exists(output_data_folder):
+        os.makedirs(output_data_folder)
+
+    train_file = open(output_train_filename, 'w')
+    test_file = open(output_test_filename, 'w')
+
+    for folder_scene in scenes_list:
+
+        # only take care of maxwell scenes
+        scene_path = os.path.join(path, folder_scene)
+
+        zones_indices = zones
+
+        # write into file
+        folder_learned_path = os.path.join(learned_folder, _filename.split('/')[1])
+
+        if not os.path.exists(folder_learned_path):
+            os.makedirs(folder_learned_path)
+
+        file_learned_path = os.path.join(folder_learned_path, folder_scene + '.csv')
+
+        with open(file_learned_path, 'w') as f:
+            for i in _zones:
+                f.write(str(i) + ';')
+
+        for id_zone, index_folder in enumerate(zones_indices):
+
+            index_str = str(index_folder)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+            current_zone_folder = "zone" + index_str
+
+            zone_path = os.path.join(scene_path, current_zone_folder)
+
+            # if custom normalization choices then we use svd values not already normalized
+            if _custom:
+                data_filename = _feature + "_svd" + generic_output_file_svd
+            else:
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
+
+            data_file_path = os.path.join(zone_path, data_filename)
+
+            # getting number of line and read randomly lines
+            f = open(data_file_path)
+            lines = f.readlines()
+
+            num_lines = len(lines)
+
+            lines_indexes = np.arange(num_lines)
+            random.shuffle(lines_indexes)
+
+            path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+            counter = 0
+            # check if user select current scene and zone to be part of training data set
+            for index in lines_indexes:
+
+                image_index = int(lines[index].split(';')[0])
+                percent = counter / num_lines
+
+                if image_index % _step == 0:
+                    line = construct_new_line(path_seuil, _interval, lines[index], _choice, _each, _norm)
+
+                    if id_zone in _zones and folder_scene in _scenes and percent <= _percent:
+                        train_file.write(line)
+                    else:
+                        test_file.write(line)
+
+                counter += 1
+
+            f.close()
+
+    train_file.close()
+    test_file.close()
+
+
+def main():
+
+    # getting all params
+    parser = argparse.ArgumentParser(description="Generate data for model using correlation matrix information from data")
+
+    parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
+    parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
+    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
+    parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
+    parser.add_argument('--zones', type=str, help='Zones indices to use for training data set')
+    parser.add_argument('--percent', type=float, help='Percent of data use for train and test dataset (by default 1)', default=1.0)
+    parser.add_argument('--step', type=int, help='Photo step to keep for build datasets', default=1)
+    parser.add_argument('--each', type=int, help='Each features to keep from interval', default=1)
+    parser.add_argument('--renderer', type=str, help='Renderer choice in order to limit scenes used', choices=renderer_choices, default='all')
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
+
+    args = parser.parse_args()
+
+    p_filename = args.output
+    p_interval = list(map(int, args.interval.split(',')))
+    p_kind     = args.kind
+    p_feature  = args.feature
+    p_scenes   = args.scenes.split(',')
+    p_zones    = list(map(int, args.zones.split(',')))
+    p_percent  = args.percent
+    p_step     = args.step
+    p_each     = args.each
+    p_renderer = args.renderer
+    p_custom   = args.custom
+
+    # list all possibles choices of renderer
+    scenes_list = dt.get_renderer_scenes_names(p_renderer)
+    scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
+
+    # getting scenes from indexes user selection
+    scenes_selected = []
+
+    for scene_id in p_scenes:
+        index = scenes_indices.index(scene_id.strip())
+        scenes_selected.append(scenes_list[index])
+
+    # find min max value if necessary to renormalize data
+    if p_custom:
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
+
+        # write new file to save
+        if not os.path.exists(custom_min_max_folder):
+            os.makedirs(custom_min_max_folder)
+
+        min_max_folder_path = os.path.join(os.path.dirname(__file__), custom_min_max_folder)
+        min_max_filename_path = os.path.join(min_max_folder_path, p_custom)
+
+        with open(min_max_filename_path, 'w') as f:
+            f.write(str(min_value_interval) + '\n')
+            f.write(str(max_value_interval) + '\n')
+
+    # create database using img folder (generate first time only)
+    generate_data_model(p_filename, p_interval, p_kind, p_feature, scenes_selected, p_zones, p_percent, p_step, p_each, p_custom)
+
+if __name__== "__main__":
+    main()

generate/generate_data_model_corr_random.py

@@ -0,0 +1,379 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+import pandas as pd
+import subprocess
+import random
+
+# image processing imports
+from PIL import Image
+
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+from data_attributes import get_image_features
+
+
+# getting configuration information
+learned_folder          = cfg.learned_zones_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes variables
+all_scenes_list         = cfg.scenes_names
+all_scenes_indices      = cfg.scenes_indices
+
+renderer_choices        = cfg.renderer_choices
+normalization_choices   = cfg.normalization_choices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+features_choices        = cfg.features_choices_labels
+output_data_folder      = cfg.output_data_folder
+custom_min_max_folder   = cfg.min_max_custom_folder
+min_max_ext             = cfg.min_max_filename_extension
+
+generic_output_file_svd = '_random.csv'
+
+min_value_interval      = sys.maxsize
+max_value_interval      = 0
+
+
+def construct_new_line(path_seuil, indices, line, choice, norm):
+
+    # increase indices values by one to avoid label
+    f = lambda x : x + 1
+    indices = f(indices)
+
+    line_data = np.array(line.split(';'))
+    seuil = line_data[0]
+    features = line_data[indices]
+    features = features.astype('float32')
+
+    # TODO : check if it's always necessary to do that (loss of information for svd)
+    if norm:
+        if choice == 'svdne':
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
+        if choice == 'svdn':
+            features = utils.normalize_arr(features)
+
+    with open(path_seuil, "r") as seuil_file:
+        seuil_learned = int(seuil_file.readline().strip())
+
+    if seuil_learned > int(seuil):
+        line = '1'
+    else:
+        line = '0'
+
+    for val in features:
+        line += ';'
+        line += str(val)
+    line += '\n'
+
+    return line
+
+def get_min_max_value_interval(_scenes_list, _indices, _feature):
+
+    global min_value_interval, max_value_interval
+
+    # increase indices values by one to avoid label
+    f = lambda x : x + 1
+    _indices = f(_indices)
+
+    scenes = os.listdir(path)
+
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    for folder_scene in scenes:
+
+        # only take care of maxwell scenes
+        if folder_scene in _scenes_list:
+
+            scene_path = os.path.join(path, folder_scene)
+
+            zones_folder = []
+            # create zones list
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+                zones_folder.append("zone"+index_str)
+
+            for zone_folder in zones_folder:
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                # if custom normalization choices then we use svd values not already normalized
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+
+                data_file_path = os.path.join(zone_path, data_filename)
+
+                # getting number of line and read randomly lines
+                f = open(data_file_path)
+                lines = f.readlines()
+
+                # check if user select current scene and zone to be part of training data set
+                for line in lines:
+
+                    line_data = np.array(line.split(';'))
+
+                    features = line_data[[_indices]]
+                    features = [float(m) for m in features]
+
+                    min_value = min(features)
+                    max_value = max(features)
+
+                    if min_value < min_value_interval:
+                        min_value_interval = min_value
+
+                    if max_value > max_value_interval:
+                        max_value_interval = max_value
+
+
+def generate_data_model(_scenes_list, _filename, _interval, _choice, _feature, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _custom = False):
+
+    output_train_filename = _filename + ".train"
+    output_test_filename = _filename + ".test"
+
+    if not '/' in output_train_filename:
+        raise Exception("Please select filename with directory path to save data. Example : data/dataset")
+
+    # create path if not exists
+    if not os.path.exists(output_data_folder):
+        os.makedirs(output_data_folder)
+
+    train_file_data = []
+    test_file_data  = []
+
+    for folder_scene in _scenes_list:
+
+        scene_path = os.path.join(path, folder_scene)
+
+        zones_indices = zones
+
+        # shuffle list of zones (=> randomly choose zones)
+        # only in random mode
+        if _random:
+            random.shuffle(zones_indices)
+
+        # store zones learned
+        learned_zones_indices = zones_indices[:_nb_zones]
+
+        # write into file
+        folder_learned_path = os.path.join(learned_folder, _filename.split('/')[1])
+
+        if not os.path.exists(folder_learned_path):
+            os.makedirs(folder_learned_path)
+
+        file_learned_path = os.path.join(folder_learned_path, folder_scene + '.csv')
+
+        with open(file_learned_path, 'w') as f:
+            for i in learned_zones_indices:
+                f.write(str(i) + ';')
+
+        for id_zone, index_folder in enumerate(zones_indices):
+
+            index_str = str(index_folder)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+            current_zone_folder = "zone" + index_str
+
+            zone_path = os.path.join(scene_path, current_zone_folder)
+
+            # if custom normalization choices then we use svd values not already normalized
+            if _custom:
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+            else:
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
+
+            data_file_path = os.path.join(zone_path, data_filename)
+
+            # getting number of line and read randomly lines
+            f = open(data_file_path)
+            lines = f.readlines()
+
+            num_lines = len(lines)
+
+            # randomly shuffle image
+            if _random:
+                random.shuffle(lines)
+
+            path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+            counter = 0
+            # check if user select current scene and zone to be part of training data set
+            for data in lines:
+
+                percent = counter / num_lines
+                image_index = int(data.split(';')[0])
+
+                if image_index % _step == 0:
+                    line = construct_new_line(path_seuil, _interval, data, _choice, _custom)
+
+                    if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
+                        train_file_data.append(line)
+                    else:
+                        test_file_data.append(line)
+
+                counter += 1
+
+            f.close()
+
+    train_file = open(output_train_filename, 'w')
+    test_file = open(output_test_filename, 'w')
+
+    for line in train_file_data:
+        train_file.write(line)
+
+    for line in test_file_data:
+        test_file.write(line)
+
+    train_file.close()
+    test_file.close()
+
+
+def main():
+
+    # getting all params
+    parser = argparse.ArgumentParser(description="Generate data for model using correlation matrix information from data")
+
+    parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
+    parser.add_argument('--n', type=int, help='Number of features wanted')
+    parser.add_argument('--highest', type=int, help='Specify if highest or lowest values are wishes', choices=[0, 1])
+    parser.add_argument('--label', type=int, help='Specify if label correlation is used or not', choices=[0, 1])
+    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
+    parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
+    parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set')
+    parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
+    parser.add_argument('--percent', type=float, help='Percent of data use for train and test dataset (by default 1)')
+    parser.add_argument('--step', type=int, help='Photo step to keep for build datasets', default=1)
+    parser.add_argument('--renderer', type=str, help='Renderer choice in order to limit scenes used', choices=renderer_choices, default='all')
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
+
+    args = parser.parse_args()
+
+    p_filename = args.output
+    p_n        = args.n
+    p_highest  = args.highest
+    p_label    = args.label
+    p_kind     = args.kind
+    p_feature  = args.feature
+    p_scenes   = args.scenes.split(',')
+    p_nb_zones = args.nb_zones
+    p_random   = args.random
+    p_percent  = args.percent
+    p_step     = args.step
+    p_renderer = args.renderer
+    p_custom   = args.custom
+
+    # list all possibles choices of renderer
+    scenes_list = dt.get_renderer_scenes_names(p_renderer)
+    scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
+
+    # getting scenes from indexes user selection
+    scenes_selected = []
+
+    for scene_id in p_scenes:
+        index = scenes_indices.index(scene_id.strip())
+        scenes_selected.append(scenes_list[index])
+
+    # Get indices to keep from correlation information
+    # compute temp data file to get correlation information
+    temp_filename = 'temp'
+    temp_filename_path = os.path.join(cfg.output_data_folder, temp_filename)
+
+    cmd = ['python', 'generate_data_model_random.py',
+            '--output', temp_filename_path,
+            '--interval', '0, 200',
+            '--kind', p_kind,
+            '--feature', p_feature,
+            '--scenes', args.scenes,
+            '--nb_zones', str(16),
+            '--random', str(int(p_random)),
+            '--percent', str(p_percent),
+            '--step', str(p_step),
+            '--each', str(1),
+            '--renderer', p_renderer,
+            '--custom', temp_filename + min_max_ext]
+
+    subprocess.Popen(cmd).wait()
+
+    temp_data_file_path = temp_filename_path + '.train'
+    df = pd.read_csv(temp_data_file_path, sep=';', header=None)
+
+    indices = []
+
+    # compute correlation matrix from whole data scenes of renderer (using or not label column)
+    if p_label:
+
+        # compute pearson correlation between features and label
+        corr = df.corr()
+
+        features_corr = []
+
+        for id_row, row in enumerate(corr):
+            for id_col, val in enumerate(corr[row]):
+                if id_col == 0 and id_row != 0:
+                    features_corr.append(abs(val))
+
+    else:
+        df = df.drop(df.columns[[0]], axis=1)
+
+        # compute pearson correlation between features using only features
+        corr = df[1:200].corr()
+
+        features_corr = []
+
+        for id_row, row in enumerate(corr):
+            correlation_score = 0
+            for id_col, val in enumerate(corr[row]):
+                if id_col != id_row:
+                    correlation_score += abs(val)
+
+            features_corr.append(correlation_score)
+
+    # find `n` min or max indices to keep
+    if p_highest:
+        indices = utils.get_indices_of_highest_values(features_corr, p_n)
+    else:
+        indices = utils.get_indices_of_lowest_values(features_corr, p_n)
+
+    indices = np.sort(indices)
+
+    # save indices found
+    if not os.path.exists(cfg.correlation_indices_folder):
+        os.makedirs(cfg.correlation_indices_folder)
+
+    indices_file_path = os.path.join(cfg.correlation_indices_folder, p_filename.replace(cfg.output_data_folder + '/', '') + '.csv')
+
+    with open(indices_file_path, 'w') as f:
+        for i in indices:
+            f.write(str(i) + ';')
+
+    # find min max value if necessary to renormalize data from `n` indices found
+    if p_custom:
+        get_min_max_value_interval(scenes_list, indices, p_feature)
+
+        # write new file to save
+        if not os.path.exists(custom_min_max_folder):
+            os.makedirs(custom_min_max_folder)
+
+        min_max_current_filename = p_filename.replace(cfg.output_data_folder + '/', '').replace('deep_keras_', '') + min_max_filename
+        min_max_filename_path = os.path.join(custom_min_max_folder, min_max_current_filename)
+
+        print(min_max_filename_path)
+        with open(min_max_filename_path, 'w') as f:
+            f.write(str(min_value_interval) + '\n')
+            f.write(str(max_value_interval) + '\n')
+
+    # create database using img folder (generate first time only)
+    generate_data_model(scenes_list, p_filename, indices, p_kind, p_feature, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_custom)
+
+if __name__== "__main__":
+    main()

generate/generate_data_model_random.py

@@ -0,0 +1,299 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+import pandas as pd
+import random
+
+# image processing imports
+from PIL import Image
+
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+from data_attributes import get_image_features
+
+
+# getting configuration information
+learned_folder          = cfg.learned_zones_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes variables
+all_scenes_list         = cfg.scenes_names
+all_scenes_indices      = cfg.scenes_indices
+
+normalization_choices   = cfg.normalization_choices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+renderer_choices        = cfg.renderer_choices
+features_choices        = cfg.features_choices_labels
+output_data_folder      = cfg.output_data_folder
+custom_min_max_folder   = cfg.min_max_custom_folder
+min_max_ext             = cfg.min_max_filename_extension
+
+generic_output_file_svd = '_random.csv'
+
+min_value_interval      = sys.maxsize
+max_value_interval      = 0
+
+def construct_new_line(path_seuil, interval, line, choice, each, norm):
+    begin, end = interval
+
+    line_data = line.split(';')
+    seuil = line_data[0]
+    features = line_data[begin+1:end+1]
+
+    # keep only if modulo result is 0 (keep only each wanted values)
+    features = [float(m) for id, m in enumerate(features) if id % each == 0]
+
+    # TODO : check if it's always necessary to do that (loss of information for svd)
+    if norm:
+
+        if choice == 'svdne':
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
+        if choice == 'svdn':
+            features = utils.normalize_arr(features)
+
+    with open(path_seuil, "r") as seuil_file:
+        seuil_learned = int(seuil_file.readline().strip())
+
+    if seuil_learned > int(seuil):
+        line = '1'
+    else:
+        line = '0'
+
+    for val in features:
+        line += ';'
+        line += str(val)
+    line += '\n'
+
+    return line
+
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
+
+    global min_value_interval, max_value_interval
+
+    scenes = os.listdir(path)
+
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    for folder_scene in scenes:
+
+        # only take care of maxwell scenes
+        if folder_scene in _scenes_list:
+
+            scene_path = os.path.join(path, folder_scene)
+
+            zones_folder = []
+            # create zones list
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+                zones_folder.append("zone"+index_str)
+
+            for zone_folder in zones_folder:
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                # if custom normalization choices then we use svd values not already normalized
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+
+                data_file_path = os.path.join(zone_path, data_filename)
+
+                # getting number of line and read randomly lines
+                f = open(data_file_path)
+                lines = f.readlines()
+
+                # check if user select current scene and zone to be part of training data set
+                for line in lines:
+
+                    begin, end = _interval
+
+                    line_data = line.split(';')
+
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
+
+                    min_value = min(features)
+                    max_value = max(features)
+
+                    if min_value < min_value_interval:
+                        min_value_interval = min_value
+
+                    if max_value > max_value_interval:
+                        max_value_interval = max_value
+
+
+def generate_data_model(_scenes_list, _filename, _interval, _choice, _feature, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
+
+    output_train_filename = _filename + ".train"
+    output_test_filename = _filename + ".test"
+
+    if not '/' in output_train_filename:
+        raise Exception("Please select filename with directory path to save data. Example : data/dataset")
+
+    # create path if not exists
+    if not os.path.exists(output_data_folder):
+        os.makedirs(output_data_folder)
+
+    train_file_data = []
+    test_file_data  = []
+
+    for folder_scene in _scenes_list:
+
+        scene_path = os.path.join(path, folder_scene)
+
+        zones_indices = zones
+
+        # shuffle list of zones (=> randomly choose zones)
+        # only in random mode
+        if _random:
+            random.shuffle(zones_indices)
+
+        # store zones learned
+        learned_zones_indices = zones_indices[:_nb_zones]
+
+        # write into file
+        folder_learned_path = os.path.join(learned_folder, _filename.split('/')[1])
+
+        if not os.path.exists(folder_learned_path):
+            os.makedirs(folder_learned_path)
+
+        file_learned_path = os.path.join(folder_learned_path, folder_scene + '.csv')
+
+        with open(file_learned_path, 'w') as f:
+            for i in learned_zones_indices:
+                f.write(str(i) + ';')
+
+        for id_zone, index_folder in enumerate(zones_indices):
+
+            index_str = str(index_folder)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+            current_zone_folder = "zone" + index_str
+
+            zone_path = os.path.join(scene_path, current_zone_folder)
+
+            # if custom normalization choices then we use svd values not already normalized
+            if _custom:
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+            else:
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
+
+            data_file_path = os.path.join(zone_path, data_filename)
+
+            # getting number of line and read randomly lines
+            f = open(data_file_path)
+            lines = f.readlines()
+
+            num_lines = len(lines)
+
+            # randomly shuffle image
+            if _random:
+                random.shuffle(lines)
+
+            path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+            counter = 0
+            # check if user select current scene and zone to be part of training data set
+            for data in lines:
+
+                percent = counter / num_lines
+                image_index = int(data.split(';')[0])
+
+                if image_index % _step == 0:
+                    line = construct_new_line(path_seuil, _interval, data, _choice, _each, _custom)
+
+                    if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
+                        train_file_data.append(line)
+                    else:
+                        test_file_data.append(line)
+
+                counter += 1
+
+            f.close()
+
+    train_file = open(output_train_filename, 'w')
+    test_file = open(output_test_filename, 'w')
+
+    for line in train_file_data:
+        train_file.write(line)
+
+    for line in test_file_data:
+        test_file.write(line)
+
+    train_file.close()
+    test_file.close()
+
+
+def main():
+
+    # getting all params
+    parser = argparse.ArgumentParser(description="Generate data for model using correlation matrix information from data")
+
+    parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
+    parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
+    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
+    parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
+    parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set')
+    parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
+    parser.add_argument('--percent', type=float, help='Percent of data use for train and test dataset (by default 1)')
+    parser.add_argument('--step', type=int, help='Photo step to keep for build datasets', default=1)
+    parser.add_argument('--each', type=int, help='Each features to keep from interval', default=1)
+    parser.add_argument('--renderer', type=str, help='Renderer choice in order to limit scenes used', choices=renderer_choices, default='all')
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
+
+    args = parser.parse_args()
+
+    p_filename = args.output
+    p_interval = list(map(int, args.interval.split(',')))
+    p_kind     = args.kind
+    p_feature  = args.feature
+    p_scenes   = args.scenes.split(',')
+    p_nb_zones = args.nb_zones
+    p_random   = args.random
+    p_percent  = args.percent
+    p_step     = args.step
+    p_each     = args.each
+    p_renderer = args.renderer
+    p_custom   = args.custom
+
+
+    # list all possibles choices of renderer
+    scenes_list = dt.get_renderer_scenes_names(p_renderer)
+    scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
+
+    # getting scenes from indexes user selection
+    scenes_selected = []
+
+    for scene_id in p_scenes:
+        index = scenes_indices.index(scene_id.strip())
+        scenes_selected.append(scenes_list[index])
+
+    # find min max value if necessary to renormalize data
+    if p_custom:
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
+
+        # write new file to save
+        if not os.path.exists(custom_min_max_folder):
+            os.makedirs(custom_min_max_folder)
+
+        min_max_filename_path = os.path.join(custom_min_max_folder, p_custom)
+
+        with open(min_max_filename_path, 'w') as f:
+            f.write(str(min_value_interval) + '\n')
+            f.write(str(max_value_interval) + '\n')
+
+    # create database using img folder (generate first time only)
+    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_feature, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
+
+if __name__== "__main__":
+    main()

generate/generate_data_model_random_center.py

@@ -0,0 +1,310 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+import pandas as pd
+import random
+
+# image processing imports
+from PIL import Image
+
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+from data_attributes import get_image_features
+
+
+# getting configuration information
+learned_folder          = cfg.learned_zones_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes variables
+all_scenes_list         = cfg.scenes_names
+all_scenes_indices      = cfg.scenes_indices
+
+normalization_choices   = cfg.normalization_choices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+renderer_choices        = cfg.renderer_choices
+features_choices        = cfg.features_choices_labels
+output_data_folder      = cfg.output_data_folder
+custom_min_max_folder   = cfg.min_max_custom_folder
+min_max_ext             = cfg.min_max_filename_extension
+
+generic_output_file_svd = '_random.csv'
+
+min_value_interval      = sys.maxsize
+max_value_interval      = 0
+abs_gap_data            = 150
+
+
+def construct_new_line(seuil_learned, interval, line, choice, each, norm):
+    begin, end = interval
+
+    line_data = line.split(';')
+    seuil = line_data[0]
+    features = line_data[begin+1:end+1]
+
+    # keep only if modulo result is 0 (keep only each wanted values)
+    features = [float(m) for id, m in enumerate(features) if id % each == 0]
+
+    # TODO : check if it's always necessary to do that (loss of information for svd)
+    if norm:
+
+        if choice == 'svdne':
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
+        if choice == 'svdn':
+            features = utils.normalize_arr(features)
+
+    if seuil_learned > int(seuil):
+        line = '1'
+    else:
+        line = '0'
+
+    for val in features:
+        line += ';'
+        line += str(val)
+    line += '\n'
+
+    return line
+
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
+
+    global min_value_interval, max_value_interval
+
+    scenes = os.listdir(path)
+
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    for folder_scene in scenes:
+
+        # only take care of maxwell scenes
+        if folder_scene in _scenes_list:
+
+            scene_path = os.path.join(path, folder_scene)
+
+            zones_folder = []
+            # create zones list
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+                zones_folder.append("zone"+index_str)
+
+            for zone_folder in zones_folder:
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                # if custom normalization choices then we use svd values not already normalized
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+
+                data_file_path = os.path.join(zone_path, data_filename)
+
+                # getting number of line and read randomly lines
+                f = open(data_file_path)
+                lines = f.readlines()
+
+                # check if user select current scene and zone to be part of training data set
+                for line in lines:
+
+                    begin, end = _interval
+
+                    line_data = line.split(';')
+
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
+
+                    min_value = min(features)
+                    max_value = max(features)
+
+                    if min_value < min_value_interval:
+                        min_value_interval = min_value
+
+                    if max_value > max_value_interval:
+                        max_value_interval = max_value
+
+
+def generate_data_model(_scenes_list, _filename, _interval, _choice, _feature, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
+
+    output_train_filename = _filename + ".train"
+    output_test_filename = _filename + ".test"
+
+    if not '/' in output_train_filename:
+        raise Exception("Please select filename with directory path to save data. Example : data/dataset")
+
+    # create path if not exists
+    if not os.path.exists(output_data_folder):
+        os.makedirs(output_data_folder)
+
+    train_file_data = []
+    test_file_data  = []
+
+    for folder_scene in _scenes_list:
+
+        scene_path = os.path.join(path, folder_scene)
+
+        zones_indices = zones
+
+        # shuffle list of zones (=> randomly choose zones)
+        # only in random mode
+        if _random:
+            random.shuffle(zones_indices)
+
+        # store zones learned
+        learned_zones_indices = zones_indices[:_nb_zones]
+
+        # write into file
+        folder_learned_path = os.path.join(learned_folder, _filename.split('/')[1])
+
+        if not os.path.exists(folder_learned_path):
+            os.makedirs(folder_learned_path)
+
+        file_learned_path = os.path.join(folder_learned_path, folder_scene + '.csv')
+
+        with open(file_learned_path, 'w') as f:
+            for i in learned_zones_indices:
+                f.write(str(i) + ';')
+
+        for id_zone, index_folder in enumerate(zones_indices):
+
+            index_str = str(index_folder)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+            current_zone_folder = "zone" + index_str
+
+            zone_path = os.path.join(scene_path, current_zone_folder)
+
+            # if custom normalization choices then we use svd values not already normalized
+            if _custom:
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+            else:
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
+
+            data_file_path = os.path.join(zone_path, data_filename)
+
+            # getting number of line and read randomly lines
+            f = open(data_file_path)
+            lines = f.readlines()
+
+            num_lines = len(lines)
+
+            # randomly shuffle image
+            if _random:
+                random.shuffle(lines)
+
+            path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+            with open(path_seuil, "r") as seuil_file:
+                seuil_learned = int(seuil_file.readline().strip())
+
+            counter = 0
+            # check if user select current scene and zone to be part of training data set
+            for data in lines:
+
+                percent = counter / num_lines
+                image_index = int(data.split(';')[0])
+
+                if image_index % _step == 0:
+
+                    with open(path_seuil, "r") as seuil_file:
+                        seuil_learned = int(seuil_file.readline().strip())
+
+                    gap_threshold = abs(seuil_learned - image_index)
+
+                    # only keep data near to threshold of zone image
+                    if gap_threshold <= abs_gap_data:
+
+                        line = construct_new_line(seuil_learned, _interval, data, _choice, _each, _custom)
+
+                        if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
+                            train_file_data.append(line)
+                        else:
+                            test_file_data.append(line)
+
+                counter += 1
+
+            f.close()
+
+    train_file = open(output_train_filename, 'w')
+    test_file = open(output_test_filename, 'w')
+
+    for line in train_file_data:
+        train_file.write(line)
+
+    for line in test_file_data:
+        test_file.write(line)
+
+    train_file.close()
+    test_file.close()
+
+
+def main():
+
+    # getting all params
+    parser = argparse.ArgumentParser(description="Generate data for model using correlation matrix information from data")
+
+    parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
+    parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
+    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
+    parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
+    parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set')
+    parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
+    parser.add_argument('--percent', type=float, help='Percent of data use for train and test dataset (by default 1)')
+    parser.add_argument('--step', type=int, help='Photo step to keep for build datasets', default=1)
+    parser.add_argument('--each', type=int, help='Each features to keep from interval', default=1)
+    parser.add_argument('--renderer', type=str, help='Renderer choice in order to limit scenes used', choices=renderer_choices, default='all')
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
+
+    args = parser.parse_args()
+
+    p_filename = args.output
+    p_interval = list(map(int, args.interval.split(',')))
+    p_kind     = args.kind
+    p_feature  = args.feature
+    p_scenes   = args.scenes.split(',')
+    p_nb_zones = args.nb_zones
+    p_random   = args.random
+    p_percent  = args.percent
+    p_step     = args.step
+    p_each     = args.each
+    p_renderer = args.renderer
+    p_custom   = args.custom
+
+
+    # list all possibles choices of renderer
+    scenes_list = dt.get_renderer_scenes_names(p_renderer)
+    scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
+
+    # getting scenes from indexes user selection
+    scenes_selected = []
+
+    for scene_id in p_scenes:
+        index = scenes_indices.index(scene_id.strip())
+        scenes_selected.append(scenes_list[index])
+
+    # find min max value if necessary to renormalize data
+    if p_custom:
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
+
+        # write new file to save
+        if not os.path.exists(custom_min_max_folder):
+            os.makedirs(custom_min_max_folder)
+
+        min_max_filename_path = os.path.join(custom_min_max_folder, p_custom)
+
+        with open(min_max_filename_path, 'w') as f:
+            f.write(str(min_value_interval) + '\n')
+            f.write(str(max_value_interval) + '\n')
+
+    # create database using img folder (generate first time only)
+    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_feature, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
+
+if __name__== "__main__":
+    main()

generate/generate_data_model_random_split.py

@@ -0,0 +1,309 @@
+# main imports
+import sys, os, argparse
+import numpy as np
+import pandas as pd
+import random
+
+# image processing imports
+from PIL import Image
+
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+from data_attributes import get_image_features
+
+
+# getting configuration information
+learned_folder          = cfg.learned_zones_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes variables
+all_scenes_list         = cfg.scenes_names
+all_scenes_indices      = cfg.scenes_indices
+
+normalization_choices   = cfg.normalization_choices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+renderer_choices        = cfg.renderer_choices
+features_choices        = cfg.features_choices_labels
+output_data_folder      = cfg.output_data_folder
+custom_min_max_folder   = cfg.min_max_custom_folder
+min_max_ext             = cfg.min_max_filename_extension
+
+generic_output_file_svd = '_random.csv'
+
+min_value_interval      = sys.maxsize
+max_value_interval      = 0
+abs_gap_data            = 100
+
+
+def construct_new_line(seuil_learned, interval, line, choice, each, norm):
+    begin, end = interval
+
+    line_data = line.split(';')
+    seuil = line_data[0]
+    features = line_data[begin+1:end+1]
+
+    # keep only if modulo result is 0 (keep only each wanted values)
+    features = [float(m) for id, m in enumerate(features) if id % each == 0]
+
+    # TODO : check if it's always necessary to do that (loss of information for svd)
+    if norm:
+
+        if choice == 'svdne':
+            features = utils.normalize_arr_with_range(features, min_value_interval, max_value_interval)
+        if choice == 'svdn':
+            features = utils.normalize_arr(features)
+
+    if seuil_learned > int(seuil):
+        line = '1'
+    else:
+        line = '0'
+
+    for val in features:
+        line += ';'
+        line += str(val)
+    line += '\n'
+
+    return line
+
+def get_min_max_value_interval(_scenes_list, _interval, _feature):
+
+    global min_value_interval, max_value_interval
+
+    scenes = os.listdir(path)
+
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+    for folder_scene in scenes:
+
+        # only take care of maxwell scenes
+        if folder_scene in _scenes_list:
+
+            scene_path = os.path.join(path, folder_scene)
+
+            zones_folder = []
+            # create zones list
+            for index in zones:
+                index_str = str(index)
+                if len(index_str) < 2:
+                    index_str = "0" + index_str
+                zones_folder.append("zone"+index_str)
+
+            for zone_folder in zones_folder:
+
+                zone_path = os.path.join(scene_path, zone_folder)
+
+                # if custom normalization choices then we use svd values not already normalized
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+
+                data_file_path = os.path.join(zone_path, data_filename)
+
+                # getting number of line and read randomly lines
+                f = open(data_file_path)
+                lines = f.readlines()
+
+                # check if user select current scene and zone to be part of training data set
+                for line in lines:
+
+                    begin, end = _interval
+
+                    line_data = line.split(';')
+
+                    features = line_data[begin+1:end+1]
+                    features = [float(m) for m in features]
+
+                    min_value = min(features)
+                    max_value = max(features)
+
+                    if min_value < min_value_interval:
+                        min_value_interval = min_value
+
+                    if max_value > max_value_interval:
+                        max_value_interval = max_value
+
+
+def generate_data_model(_scenes_list, _filename, _interval, _choice, _feature, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _each=1, _custom = False):
+
+    output_train_filename = _filename + ".train"
+    output_test_filename = _filename + ".test"
+
+    if not '/' in output_train_filename:
+        raise Exception("Please select filename with directory path to save data. Example : data/dataset")
+
+    # create path if not exists
+    if not os.path.exists(output_data_folder):
+        os.makedirs(output_data_folder)
+
+    train_file_data = []
+    test_file_data  = []
+
+    for folder_scene in _scenes_list:
+
+        scene_path = os.path.join(path, folder_scene)
+
+        zones_indices = zones
+
+        # shuffle list of zones (=> randomly choose zones)
+        # only in random mode
+        if _random:
+            random.shuffle(zones_indices)
+
+        # store zones learned
+        learned_zones_indices = zones_indices[:_nb_zones]
+
+        # write into file
+        folder_learned_path = os.path.join(learned_folder, _filename.split('/')[1])
+
+        if not os.path.exists(folder_learned_path):
+            os.makedirs(folder_learned_path)
+
+        file_learned_path = os.path.join(folder_learned_path, folder_scene + '.csv')
+
+        with open(file_learned_path, 'w') as f:
+            for i in learned_zones_indices:
+                f.write(str(i) + ';')
+
+        for id_zone, index_folder in enumerate(zones_indices):
+
+            index_str = str(index_folder)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+            current_zone_folder = "zone" + index_str
+
+            zone_path = os.path.join(scene_path, current_zone_folder)
+
+            # if custom normalization choices then we use svd values not already normalized
+            if _custom:
+                data_filename = _feature + "_svd"+ generic_output_file_svd
+            else:
+                data_filename = _feature + "_" + _choice + generic_output_file_svd
+
+            data_file_path = os.path.join(zone_path, data_filename)
+
+            # getting number of line and read randomly lines
+            f = open(data_file_path)
+            lines = f.readlines()
+
+            num_lines = len(lines)
+
+            # randomly shuffle image
+            if _random:
+                random.shuffle(lines)
+
+            path_seuil = os.path.join(zone_path, seuil_expe_filename)
+
+            with open(path_seuil, "r") as seuil_file:
+                seuil_learned = int(seuil_file.readline().strip())
+
+            counter = 0
+            # check if user select current scene and zone to be part of training data set
+            for data in lines:
+
+                percent = counter / num_lines
+                image_index = int(data.split(';')[0])
+
+                if image_index % _step == 0:
+
+                    with open(path_seuil, "r") as seuil_file:
+                        seuil_learned = int(seuil_file.readline().strip())
+
+                    gap_threshold = abs(seuil_learned - image_index)
+
+                    if gap_threshold > abs_gap_data:
+
+                        line = construct_new_line(seuil_learned, _interval, data, _choice, _each, _custom)
+
+                        if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
+                            train_file_data.append(line)
+                        else:
+                            test_file_data.append(line)
+
+                counter += 1
+
+            f.close()
+
+    train_file = open(output_train_filename, 'w')
+    test_file = open(output_test_filename, 'w')
+
+    for line in train_file_data:
+        train_file.write(line)
+
+    for line in test_file_data:
+        test_file.write(line)
+
+    train_file.close()
+    test_file.close()
+
+
+def main():
+
+    # getting all params
+    parser = argparse.ArgumentParser(description="Generate data for model using correlation matrix information from data")
+
+    parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
+    parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
+    parser.add_argument('--kind', type=str, help='Kind of normalization level wished', choices=normalization_choices)
+    parser.add_argument('--feature', type=str, help='feature data choice', choices=features_choices)
+    parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
+    parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set')
+    parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
+    parser.add_argument('--percent', type=float, help='Percent of data use for train and test dataset (by default 1)')
+    parser.add_argument('--step', type=int, help='Photo step to keep for build datasets', default=1)
+    parser.add_argument('--each', type=int, help='Each features to keep from interval', default=1)
+    parser.add_argument('--renderer', type=str, help='Renderer choice in order to limit scenes used', choices=renderer_choices, default='all')
+    parser.add_argument('--custom', type=str, help='Name of custom min max file if use of renormalization of data', default=False)
+
+    args = parser.parse_args()
+
+    p_filename = args.output
+    p_interval = list(map(int, args.interval.split(',')))
+    p_kind     = args.kind
+    p_feature  = args.feature
+    p_scenes   = args.scenes.split(',')
+    p_nb_zones = args.nb_zones
+    p_random   = args.random
+    p_percent  = args.percent
+    p_step     = args.step
+    p_each     = args.each
+    p_renderer = args.renderer
+    p_custom   = args.custom
+
+
+    # list all possibles choices of renderer
+    scenes_list = dt.get_renderer_scenes_names(p_renderer)
+    scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
+
+    # getting scenes from indexes user selection
+    scenes_selected = []
+
+    for scene_id in p_scenes:
+        index = scenes_indices.index(scene_id.strip())
+        scenes_selected.append(scenes_list[index])
+
+    # find min max value if necessary to renormalize data
+    if p_custom:
+        get_min_max_value_interval(scenes_list, p_interval, p_feature)
+
+        # write new file to save
+        if not os.path.exists(custom_min_max_folder):
+            os.makedirs(custom_min_max_folder)
+
+        min_max_filename_path = os.path.join(custom_min_max_folder, p_custom)
+
+        with open(min_max_filename_path, 'w') as f:
+            f.write(str(min_value_interval) + '\n')
+            f.write(str(max_value_interval) + '\n')
+
+    # create database using img folder (generate first time only)
+    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_feature, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_each, p_custom)
+
+if __name__== "__main__":
+    main()

generate_all_noise.sh

@@ -6,16 +6,15 @@ for file in "images"/*; do
     IFS=' '
 
     image=${ADDR[1]%".png"}
+    echo ${image}
 
-
-    for noise in {"cauchy","gaussian","laplace","log_normal","mut_white","white","salt_pepper"}; do
+    for noise in {"cauchy","gaussian","laplace","log_normal","mut_white","white","salt_pepper_A","salt_pepper_B"}; do
 
         for identical in {"0","1"}; do
-
             if [ ${identical} == "1" ]; then
-                python noise_computation.py --noise ${noise} --image ${file} --n 1000 --identical ${identical} --output ${image}_${noise}.png --step 10 --all 1 &
+                python noise_computation.py --noise ${noise} --image ${file} --n 1000 --identical ${identical} --output ${image}_${noise}.png --step 10 --all 1
             else
-                python noise_computation.py --noise ${noise} --image ${file} --n 1000 --identical ${identical} --output ${image}_${noise}_color.png --step 10 --all 1 &
+                python noise_computation.py --noise ${noise} --image ${file} --n 1000 --identical ${identical} --output ${image}_${noise}_color.png --step 10 --all 1
             fi
 
         done

generate_data_model_random.py

@@ -1,306 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 14 21:02:42 2018
-
-@author: jbuisine
-"""
-
-from __future__ import print_function
-import sys, os, getopt
-import numpy as np
-import random
-import time
-import json
-
-from PIL import Image
-from ipfml import processing, metrics, utils
-
-from modules.utils import config as cfg
-from modules.utils import data as dt
-
-# getting configuration information
-zone_folder             = cfg.zone_folder
-min_max_filename        = cfg.min_max_filename_extension
-
-# define all scenes values
-all_scenes_list         = cfg.scenes_names
-all_scenes_indices      = cfg.scenes_indices
-
-choices                 = cfg.normalization_choices
-path                    = cfg.dataset_path
-zones                   = cfg.zones_indices
-seuil_expe_filename     = cfg.seuil_expe_filename
-
-metric_choices          = cfg.metric_choices_labels
-output_data_folder      = cfg.output_data_folder
-custom_min_max_folder   = cfg.min_max_custom_folder
-min_max_ext             = cfg.min_max_filename_extension
-
-calibration_folder      = 'calibration'
-generic_output_file_svd = '_random.csv'
-
-min_value_interval = sys.maxsize
-max_value_interval = 0
-
-def construct_new_line(path_seuil, interval, line, norm):
-    begin, end = interval
-
-    line_data = line.split(';')
-    seuil = line_data[0]
-    metrics = line_data[begin+1:end+1]
-
-    metrics = [float(m) for m in metrics]
-
-    # TODO : check if it's always necessary to do that (loss of information for svd)
-    if norm:
-        metrics = utils.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
-
-    with open(path_seuil, "r") as seuil_file:
-        seuil_learned = int(seuil_file.readline().strip())
-
-    if seuil_learned > int(seuil):
-        line = '1'
-    else:
-        line = '0'
-
-    for idx, val in enumerate(metrics):
-        line += ';'
-        line += str(val)
-    line += '\n'
-
-    return line
-
-def get_min_max_value_interval(_scenes_list, _filename, _interval, _choice, _color, _metric):
-
-    global min_value_interval, max_value_interval
-
-    scenes = os.listdir(path)
-
-    # remove min max file from scenes folder
-    scenes = [s for s in scenes if min_max_filename not in s]
-
-    # remove calibration mire from images
-    scenes = [s for s in scenes if calibration_folder not in s]
-
-    for id_scene, folder_scene in enumerate(scenes):
-
-        # only take care of synthesis scenes
-        if folder_scene in _scenes_list:
-
-            scene_path = os.path.join(path, folder_scene)
-
-            zones_folder = []
-            # create zones list
-            for index in zones:
-                index_str = str(index)
-                if len(index_str) < 2:
-                    index_str = "0" + index_str
-                zones_folder.append("zone"+index_str)
-
-            # shuffle list of zones (=> randomly choose zones)
-            random.shuffle(zones_folder)
-
-            for id_zone, zone_folder in enumerate(zones_folder):
-                zone_path = os.path.join(scene_path, zone_folder)
-
-                if _color:
-                    data_filename = _metric + "_color_" + _choice + generic_output_file_svd
-                else:
-                    data_filename = _metric + "_" + _choice + generic_output_file_svd
-
-                data_file_path = os.path.join(zone_path, data_filename)
-
-                # getting number of line and read randomly lines
-                f = open(data_file_path)
-                lines = f.readlines()
-
-                counter = 0
-                # check if user select current scene and zone to be part of training data set
-                for line in lines:
-
-                    begin, end = _interval
-
-                    line_data = line.split(';')
-                    metrics = line_data[begin+1:end+1]
-                    metrics = [float(m) for m in metrics]
-
-                    min_value = min(metrics)
-                    max_value = max(metrics)
-
-                    if min_value < min_value_interval:
-                        min_value_interval = min_value
-
-                    if max_value > max_value_interval:
-                        max_value_interval = max_value
-
-                    counter += 1
-
-
-def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _scenes = scenes_list, _nb_zones = 4, _percent = 1,  _random=0, _step=40, _color=False, _norm = False):
-
-    output_train_filename = _filename + ".train"
-    output_test_filename = _filename + ".test"
-
-    if not '/' in output_train_filename:
-        raise Exception("Please select filename with directory path to save data. Example : data/dataset")
-
-    # create path if not exists
-    if not os.path.exists(output_data_folder):
-        os.makedirs(output_data_folder)
-
-    scenes = os.listdir(path)
-
-    # remove min max file from scenes folder
-    scenes = [s for s in scenes if min_max_filename not in s]
-
-    train_file_data = []
-    test_file_data  = []
-
-    for id_scene, folder_scene in enumerate(scenes):
-
-        # only take care of maxwell scenes
-        if folder_scene in _scenes_list:
-
-            scene_path = os.path.join(path, folder_scene)
-
-            zones_folder = []
-            # create zones list
-            for index in zones:
-                index_str = str(index)
-                if len(index_str) < 2:
-                    index_str = "0" + index_str
-                zones_folder.append("zone"+index_str)
-
-            # shuffle list of zones (=> randomly choose zones)
-            if _random:
-                random.shuffle(zones_folder)
-
-            path_seuil = os.path.join(scene_path, seuil_expe_filename)
-
-            for id_zone, zone_folder in enumerate(zones_folder):
-                zone_path = os.path.join(scene_path, zone_folder)
-
-                if _color:
-                    data_filename = _metric + "_color_" + _choice + generic_output_file_svd
-                else:
-                    data_filename = _metric + "_" + _choice + generic_output_file_svd
-
-                data_file_path = os.path.join(zone_path, data_filename)
-
-                # getting number of line and read randomly lines
-                f = open(data_file_path)
-                lines = f.readlines()
-
-                num_lines = len(lines)
-
-                if _random:
-                    random.shuffle(lines_indexes)
-
-                counter = 0
-                # check if user select current scene and zone to be part of training data set
-                for data in lines:
-
-                    percent = counter / num_lines
-                    image_index = int(data.split(';')[0])
-
-                    if image_index % _step == 0:
-                        line = construct_new_line(path_seuil, _interval, data, _choice, _norm, _sep, _index)
-
-                        if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
-                            train_file_data.append(line)
-                        else:
-                            test_file_data.append(line)
-
-                    counter += 1
-
-                f.close()
-
-
-    train_file = open(output_train_filename, 'w')
-    test_file = open(output_test_filename, 'w')
-
-    for line in train_file_data:
-        train_file.write(line)
-
-    for line in test_file_data:
-        test_file_data.write(line)
-
-    train_file.close()
-    test_file.close()
-
-
-def main():
-
-    p_custom = False
-
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --random 0 --step 40 --color 0 --custom min_max_filename')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:i:k:s:n:p:r:s:c:c", ["help=", "output=", "interval=", "kind=", "metric=","scenes=", "nb_zones=", "percent=", "random=", "step=", "color=", "custom="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --random 0 --step 40 --color 0 --custom min_max_filename')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python generate_data_model_random.py --output xxxx --interval 0,20  --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --random 0 --step 40 --color 0 --custom min_max_filename')
-            sys.exit()
-        elif o in ("-o", "--output"):
-            p_filename = a
-        elif o in ("-i", "--interval"):
-            p_interval = list(map(int, a.split(',')))
-        elif o in ("-k", "--kind"):
-            p_kind = a
-        elif o in ("-m", "--metric"):
-            p_metric = a
-        elif o in ("-s", "--scenes"):
-            p_scenes = a.split(',')
-        elif o in ("-n", "--nb_zones"):
-            p_nb_zones = int(a)
-        elif o in ("-p", "--percent"):
-            p_percent = float(a)
-        elif o in ("-r", "--random"):
-            p_random = int(a)
-        elif o in ("-p", "--percent"):
-            p_step = int(a)
-        elif o in ("-c", "--color"):
-            p_color = int(a)
-        elif o in ("-c", "--custom"):
-            p_custom = a
-        else:
-            assert False, "unhandled option"
-
-    # list all possibles choices of renderer
-    scenes_list = dt.get_renderer_scenes_names(p_renderer)
-    scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
-
-    # getting scenes from indexes user selection
-    scenes_selected = []
-
-    for scene_id in p_scenes:
-        index = scenes_indices.index(scene_id.strip())
-        scenes_selected.append(scenes_list[index])
-
-    # find min max value if necessary to renormalize data
-    if p_custom:
-        get_min_max_value_interval(scenes_list, p_filename, p_interval, p_kind, p_color, p_metric)
-
-        # write new file to save
-        if not os.path.exists(custom_min_max_folder):
-            os.makedirs(custom_min_max_folder)
-
-        min_max_folder_path = os.path.join(os.path.dirname(__file__), custom_min_max_folder)
-        min_max_filename_path = os.path.join(min_max_folder_path, p_custom)
-
-        with open(min_max_filename_path, 'w') as f:
-            f.write(str(min_value_interval) + '\n')
-            f.write(str(max_value_interval) + '\n')
-
-    # create database using img folder (generate first time only)
-    generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_color, p_custom)
-
-if __name__== "__main__":
-    main()

generate_noise_all_curves.sh

@@ -21,26 +21,26 @@ for file in "images"/*; do
 
                         if [ ! -f "curves_pictures/${filename_prefix}0${filename_suffix}.png" ]; then
 
-                            python noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 0 --ylim "0, 0.05" --error ${error}
+                            python display/noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 0 --ylim "0, 0.05" --error ${error}
                         else
                             echo "Already generated.."
                         fi
 
                         if [ ! -f "curves_pictures/${filename_prefix}1${filename_suffix}.png" ]; then
-                            python noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 1 --ylim "0, 0.1" --error ${error}
+                            python display/noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 1 --ylim "0, 0.1" --error ${error}
                         else
                             echo "Already generated.."
                         fi
 
 
                         if [ ! -f "curves_pictures/${filename_prefix}0${filename_suffix}_color.png" ]; then
-                            python noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 0 --color 1 --ylim "0, 0.05" --error ${error}
+                            python display/noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 0 --color 1 --ylim "0, 0.05" --error ${error}
                         else
                             echo "Already generated.."
                         fi
 
                         if [ ! -f "curves_pictures/${filename_prefix}1${filename_suffix}_color.png" ]; then
-                            python noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 1 --color 1 --ylim "0, 0.1" --error ${error}
+                            python display/noise_svd_tend_visualization.py  --prefix generated/${image}/${noise} --metric ${metric} --n 1000 --mode ${mode} --interval "30, 800" --step 30 --norm 1 --color 1 --ylim "0, 0.1" --error ${error}
 
                         else
                             echo "Already generated.."

modules

@@ -0,0 +1 @@
+Subproject commit d5de038bdccaa58ff2123d5227482dc6c0ea2500

modules/noise.py

@@ -1,36 +0,0 @@
-from ipfml.filters import noise as nf
-
-
-def get_noise_result(_image, _n, _noise_choice, _identical=False, _p=None):
-    """Return image with applied noise using choice
-
-    Args:
-        image: image to convert
-        _n: importance of noise expected [1, 999]
-        _noise_choice: choise of noise filter to apply
-        _identical: specify if the distribution is the same or not for each chanel
-        _p: optional parameter for salt_pepper noise
-
-    Returns:
-        Noisy image with noise filter applied
-
-    """
-
-    noise_method = None
-    function_name = _noise_choice + '_noise'
-
-    try:
-        noise_method = getattr(nf, function_name)
-    except AttributeError:
-        raise NotImplementedError("Noise filter `{}` not implement `{}`".format(nf.__name__, function_name))
-
-
-    if _p:
-
-        if _noise_choice != 'salt_pepper':
-            raise ValueError("p parameter is only used for salt pepper noise...")
-
-        return noise_method(_image, _n, identical=_identical, p=_p)
-    else:
-        return noise_method(_image, _n, identical=_identical)
-

modules/utils/config.py

@@ -1,42 +0,0 @@
-import numpy as np
-
-zone_folder                     = "zone"
-output_data_folder              = 'data'
-threshold_map_folder            = 'threshold_map'
-models_information_folder       = 'models_info'
-saved_models_folder             = 'saved_models'
-min_max_custom_folder           = 'custom_norm'
-generated_folder                = 'generated'
-pictures_output_folder          = 'curves_pictures'
-
-csv_model_comparisons_filename  = "models_comparisons.csv"
-seuil_expe_filename             = 'seuilExpe'
-min_max_filename_extension      = "_min_max_values"
-config_filename                 = "config"
-filename_ext                    = 'png'
-default_number_of_images        = 1000
-
-models_names_list               = ["svm_model","ensemble_model","ensemble_model_v2"]
-
-# define all scenes values
-renderer_choices                = ['maxwell', 'igloo', 'cycle']
-
-scenes_names                    = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
-scenes_indices                  = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
-
-maxwell_scenes_names            = ['Appart1opt02', 'Cuisine01', 'SdbCentre', 'SdbDroite']
-maxwell_scenes_indices          = ['A', 'D', 'G', 'H']
-
-igloo_scenes_names              = ['Bureau1', 'PNDVuePlongeante']
-igloo_scenes_indices            = ['B', 'F']
-
-cycle_scenes_names              = ['EchecBas', 'Selles']
-cycle_scenes_indices            = ['E', 'I']
-
-normalization_choices           = ['svd', 'svdn', 'svdne']
-zones_indices                   = np.arange(16)
-
-metric_choices_labels           = ['lab', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2']
-
-# noise information
-noise_labels                    = ['cauchy', 'gaussian', 'laplace', 'log_normal', 'mut_white', 'salt_pepper', 'white']

modules/utils/data_type.py

@@ -1,104 +0,0 @@
-from ipfml import processing, metrics
-from modules.utils.config import *
-from PIL import Image
-from skimage import color
-
-import random
-import numpy as np
-
-_scenes_names_prefix   = '_scenes_names'
-_scenes_indices_prefix = '_scenes_indices'
-
-# store all variables from current module context
-context_vars = vars()
-
-def get_svd_data(data_type, block):
-    """
-    Method which returns the data type expected
-    """
-
-    if data_type == 'lab':
-
-        nb = random.randint(0, 10)
-
-        block_file_path = '/tmp/' + str(nb) + '_lab_img.png'
-        block.save(block_file_path)
-        data = processing.get_LAB_L_SVD_s(Image.open(block_file_path))
-
-    if data_type == 'mscn_revisited':
-
-        nb = random.randint(0, 10)
-
-        img_mscn_revisited = processing.rgb_to_mscn(block)
-
-        # save tmp as img
-        img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
-        mscn_revisited_file_path = '/tmp/' + str(nb) + '_mscn_revisited_img.png'
-        img_output.save(mscn_revisited_file_path)
-        img_block = Image.open(mscn_revisited_file_path)
-
-        # extract from temp image
-        data = metrics.get_SVD_s(img_block)
-
-    if data_type == 'mscn':
-
-        img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
-        img_mscn = processing.calculate_mscn_coefficients(img_gray, 7)
-        img_mscn_norm = processing.normalize_2D_arr(img_mscn)
-
-        img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
-
-        data = metrics.get_SVD_s(img_mscn_gray)
-
-    if data_type == 'low_bits_6':
-
-        low_bits_6 = processing.rgb_to_LAB_L_low_bits(block, 6)
-        data = metrics.get_SVD_s(low_bits_6)
-
-    if data_type == 'low_bits_5':
-
-        low_bits_5 = processing.rgb_to_LAB_L_low_bits(block, 5)
-        data = metrics.get_SVD_s(low_bits_5)
-
-    if data_type == 'low_bits_4':
-
-        low_bits_4 = processing.rgb_to_LAB_L_low_bits(block, 4)
-        data = metrics.get_SVD_s(low_bits_4)
-
-    if data_type == 'low_bits_3':
-
-        low_bits_3 = processing.rgb_to_LAB_L_low_bits(block, 3)
-        data = metrics.get_SVD_s(low_bits_3)
-
-    if data_type == 'low_bits_2':
-
-        low_bits_2 = processing.rgb_to_LAB_L_low_bits(block, 2)
-        data = metrics.get_SVD_s(low_bits_2)
-
-    if data_type == 'low_bits_4_shifted_2':
-
-        data = metrics.get_SVD_s(processing.rgb_to_LAB_L_bits(block, (3, 6)))
-
-    return data
-
-def get_renderer_scenes_indices(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_indices
-    else:
-        return context_vars[renderer_name + _scenes_indices_prefix]
-
-def get_renderer_scenes_names(renderer_name):
-
-    if renderer_name not in renderer_choices:
-        raise ValueError("Unknown renderer name")
-
-    if renderer_name == 'all':
-        return scenes_names
-    else:
-        return context_vars[renderer_name + _scenes_names_prefix]
-
-

noise_computation.py

@@ -1,14 +1,23 @@
-from ipfml.filters import noise as nf
-import sys, os, getopt
+# main imports
+import sys, os, argparse
+
+# image processing imports
 from PIL import Image
+from ipfml.filters import noise as nf
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
 
-from modules.utils import config as cfg
-from modules import noise
+import custom_config as cfg
+from data_attributes import get_noise_result
 
+
+# other variables
 noise_list       = cfg.noise_labels
 generated_folder = cfg.generated_folder
 filename_ext     = cfg.filename_ext
 
+
 def generate_noisy_image(p_image, p_n, p_noise, p_identical, p_output, p_param):
 
     image_folder = p_image.filename.split('/')[-1].replace('.' + filename_ext, '')
@@ -25,7 +34,7 @@ def generate_noisy_image(p_image, p_n, p_noise, p_identical, p_output, p_param):
 
     if not os.path.exists(output_image_path):
 
-        noisy_image = noise.get_noise_result(p_image, p_n, _noise_choice=p_noise, _identical=p_identical, _p=p_param)
+        noisy_image = get_noise_result(p_image, p_n, _noise_choice=p_noise, _identical=p_identical, _p=p_param)
         noisy_image = Image.fromarray(noisy_image)
 
         noisy_image.save(output_image_path)
@@ -38,63 +47,43 @@ def generate_noisy_image(p_image, p_n, p_noise, p_identical, p_output, p_param):
 
 def main():
 
-    # by default..
-    p_step = 1
-    p_param = None
-    p_all = False
-
-    if len(sys.argv) < 1:
-        print('python noise_computation.py --noise xxxx --image path/to/image.png --n 100 --identical 0 --output image_name --step 10 --all 1 --p 0.1')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "h:n:i:n:i:o:a:p", ["help=", "noise=", "image=", "n=", "identical=", "output=", "step=", "all=", "p="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python noise_computation.py --noise xxxx --image path/to/image.png --n 100 --identical 0 --output image_name --step 10 --all 1 --p 0.1')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python noise_computation.py --noise xxxx --image path/to/image.png --n 100 --identical 0 --output image_name --step 10 --all 1 --p 0.1')
-            sys.exit()
-        elif o in ("-n", "--noise"):
-            p_noise = a
-
-            if not p_noise in noise_list:
-                assert False, "Unknow noise parameter %s, %s " % (p_noise, noise_list)
-
-        elif o in ("-i", "--image"):
-            p_image_path = a
-        elif o in ("-n", "--n"):
-            p_n = int(a)
-
-        elif o in ("-i", "--identical"):
-            p_identical = int(a)
-        elif o in ("-s", "--step"):
-            p_step = int(a)
-        elif o in ("-o", "--output"):
-            p_output = a
-        elif o in ("-a", "--all"):
-            p_all = int(a)
-        elif o in ("-p", "--p"):
-            p_param = float(a)
-        else:
-            assert False, "unhandled option"
-
-    img = Image.open(p_image_path)
-
-    if p_all:
-
-        split_output = p_output.split('.')
-
-        for i in range(1, p_n):
-
-            if i % p_step == 0:
-                p_filename = split_output[0] + "_" + str(i) + "." + filename_ext
-
-                generate_noisy_image(img, i, p_noise, p_identical, p_filename, p_param)
+    parser = argparse.ArgumentParser(description="Compute noise on specific image")
+
+    parser.add_argument('--noise', type=str, help='Noise choice to apply', choices=cfg.noise_labels)
+    parser.add_argument('--image', type=str, help='image path')
+    parser.add_argument('--n', type=int, help='Number of images')
+    parser.add_argument('--identical', type=int, help='Use of color or grey level', default=0)
+    parser.add_argument('--output', type=int, help='Output image name', default=0)
+    parser.add_argument('--step', type=int, help='Step of image indices to keep', default=1)
+    parser.add_argument('--all', type=int, help='Use of all image until `n` or just `n`', default=1)
+    parser.add_argument('--p', type=float, help='Distribution to use for noise', default=0.1)
+
+    args = parser.parse_args()
+
+    param_noise     = args.noise
+    param_image     = args.image
+    param_n         = args.n
+    param_identical = args.identical
+    param_output    = args.output
+    param_step      = args.step
+    param_all       = args.all
+    param_p         = args.p
+
+    img = Image.open(param_image)
+
+    if param_all:
+
+        split_output = param_output.split('.')
+
+        for i in range(1, param_n):
+
+            if i % param_step == 0:
+                param_filename = split_output[0] + "_" + str(i) + "." + filename_ext
+
+                generate_noisy_image(img, i, param_noise, param_identical, param_filename, param_p)
 
     else:
-        generate_noisy_image(img, p_n, p_noise, p_identical, p_output, p_param)
+        generate_noisy_image(img, param_n, param_noise, param_identical, param_output, param_p)
 
 if __name__== "__main__":
     main()

noise_svd_tend_visualization.py

@@ -1,224 +0,0 @@
-import sys, os, getopt
-from PIL import Image
-
-from ipfml import processing, utils
-import ipfml.iqa.fr as fr_iqa
-
-from modules.utils import config as cfg
-from modules.utils import data_type as dt
-from modules import noise
-import numpy as np
-
-import matplotlib.pyplot as plt
-plt.style.use('ggplot')
-
-noise_list            = cfg.noise_labels
-generated_folder      = cfg.generated_folder
-filename_ext          = cfg.filename_ext
-metric_choices        = cfg.metric_choices_labels
-normalization_choices = cfg.normalization_choices
-pictures_folder       = cfg.pictures_output_folder
-
-step_picture          = 10
-
-error_data_choices    = ['mae', 'mse', 'ssim', 'psnr']
-
-
-def get_error_distance(p_error, y_true, y_test):
-
-    noise_method = None
-    function_name = p_error
-
-    try:
-        error_method = getattr(fr_iqa, function_name)
-    except AttributeError:
-        raise NotImplementedError("Error method `{}` not implement `{}`".format(fr_iqa.__name__, function_name))
-
-    return error_method(y_true, y_test)
-
-
-def main():
-
-    # default values
-    p_step = 1
-    p_color = 0
-    p_norm = 0
-    p_ylim = (0, 1)
-
-    max_value_svd = 0
-    min_value_svd = sys.maxsize
-
-    if len(sys.argv) <= 1:
-        print('python noise_svd_tend_visualization.py --prefix generated/prefix/noise --metric lab --mode svdn --n 300 --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1" --error mae')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "h:p:m:m:n:i:s:c:n:y:e", ["help=", "prefix=", "metric=", "mode=", "n=", "interval=", "step=", "color=", "norm=", "ylim=", "error="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python noise_svd_tend_visualization.py --prefix generated/prefix/noise --metric lab --mode svdn --n 300 --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1" --error mae')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python noise_svd_tend_visualization.py --prefix generated/prefix/noise --metric lab --mode svdn --n 300 --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1" --error MAE')
-            sys.exit()
-        elif o in ("-p", "--prefix"):
-            p_path = a
-        elif o in ("-m", "--mode"):
-            p_mode = a
-
-            if not p_mode in normalization_choices:
-                assert False, "Unknown normalization choice, %s" % normalization_choices
-
-        elif o in ("-m", "--metric"):
-            p_metric = a
-
-            if not p_metric in metric_choices:
-                assert False, "Unknown metric choice, %s" % metric_choices
-
-        elif o in ("-n", "--n"):
-            p_n = int(a)
-        elif o in ("-n", "--norm"):
-            p_norm = int(a)
-        elif o in ("-c", "--color"):
-            p_color = int(a)
-        elif o in ("-i", "--interval"):
-            p_interval = list(map(int, a.split(',')))
-        elif o in ("-s", "--step"):
-            p_step = int(a)
-        elif o in ("-y", "--ylim"):
-            p_ylim = list(map(float, a.split(',')))
-        elif o in ("-e", "--error"):
-            p_error = a
-
-            if p_error not in error_data_choices:
-                assert False, "Unknow error choice to display %s" % error_data_choices
-        else:
-            assert False, "unhandled option"
-
-
-    p_prefix = p_path.split('/')[1].replace('_', '')
-    noise_name = p_path.split('/')[2]
-
-    if p_color:
-        file_path = os.path.join(p_path, p_prefix + "_" + noise_name + "_color_{}." + filename_ext)
-    else:
-        file_path = os.path.join(p_path, p_prefix + "_" + noise_name + "_{}." + filename_ext)
-
-    begin, end = p_interval
-    all_svd_data = []
-
-    svd_data = []
-    image_indices = []
-
-    noise_indices = range(1, p_n)[::-1]
-
-    # get all data from images
-    for i in noise_indices:
-
-        if i % step_picture == 0:
-
-            image_path = file_path.format(str(i))
-
-            img = Image.open(image_path)
-
-            svd_values = dt.get_svd_data(p_metric, img)
-
-            if p_norm:
-                svd_values = svd_values[begin:end]
-
-            all_svd_data.append(svd_values)
-
-            # update min max values
-            min_value = svd_values.min()
-            max_value = svd_values.max()
-
-            if min_value < min_value_svd:
-                min_value_svd = min_value
-
-            if max_value > max_value_svd:
-                max_value_svd = max_value
-
-        print('%.2f%%' % ((p_n - i + 1) / p_n * 100))
-        sys.stdout.write("\033[F")
-
-    previous_data = []
-    error_data = [0.]
-
-    for id, data in enumerate(all_svd_data):
-
-        current_id = (p_n - ((id + 1) * 10))
-
-        if current_id % p_step == 0:
-
-            current_data = data
-
-            if p_mode == 'svdn':
-                current_data = utils.normalize_arr(current_data)
-
-            if p_mode == 'svdne':
-                current_data = utils.normalize_arr_with_range(current_data, min_value_svd, max_value_svd)
-
-            svd_data.append(current_data)
-            image_indices.append(current_id)
-
-            # use of whole image data for computation of ssim or psnr
-            if p_error == 'ssim' or p_error == 'psnr':
-                image_path = file_path.format(str(current_id))
-                current_data = np.asarray(Image.open(image_path))
-
-            if len(previous_data) > 0:
-
-                current_error = get_error_distance(p_error, previous_data, current_data)
-                error_data.append(current_error)
-
-            if len(previous_data) == 0:
-                previous_data = current_data
-
-    # display all data using matplotlib (configure plt)
-    gridsize = (3, 2)
-
-    # fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(30, 22))
-    fig = plt.figure(figsize=(30, 22))
-    ax1 = plt.subplot2grid(gridsize, (0, 0), colspan=2, rowspan=2)
-    ax2 = plt.subplot2grid(gridsize, (2, 0), colspan=2)
-
-    ax1.set_title(p_prefix  + ', ' + noise_name + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + p_metric + ' metric, step ' + str(p_step) + ' normalization ' + p_mode)
-    ax1.set_label('Importance of noise [1, 999]')
-    ax1.set_xlabel('Vector features')
-
-    for id, data in enumerate(svd_data):
-
-        p_label = p_prefix + str(image_indices[id]) + " | " + p_error + ": " + str(error_data[id])
-        ax1.plot(data, label=p_label)
-
-    ax1.legend(bbox_to_anchor=(0.8, 1), loc=2, borderaxespad=0.2, fontsize=12)
-
-    if not p_norm:
-        ax1.set_xlim(begin, end)
-
-    # adapt ylim
-    y_begin, y_end = p_ylim
-    ax1.set_ylim(y_begin, y_end)
-
-    output_filename = p_prefix + "_" + noise_name + "_1_to_" + str(p_n) + "_B" + str(begin) + "_E" + str(end) + "_" + p_metric + "_S" + str(p_step) + "_norm" + str(p_norm )+  "_" + p_mode + "_" + p_error
-
-    if p_color:
-        output_filename = output_filename + '_color'
-
-    ax2.set_title(p_error + " information for : " + p_prefix  + ', ' + noise_name + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + p_metric + ' metric, step ' + str(p_step) + ', normalization ' + p_mode)
-    ax2.set_ylabel(p_error + ' error')
-    ax2.set_xlabel('Number of samples per pixels')
-    ax2.set_xticks(range(len(image_indices)))
-    ax2.set_xticklabels(image_indices)
-    ax2.plot(error_data)
-
-    print("Generation of output figure... %s" % output_filename)
-    output_path = os.path.join(pictures_folder, output_filename)
-
-    if not os.path.exists(pictures_folder):
-        os.makedirs(pictures_folder)
-
-    fig.savefig(output_path, dpi=(200))
-
-if __name__== "__main__":
-    main()

noise_svd_visualization.py

@@ -1,169 +0,0 @@
-import sys, os, getopt
-from PIL import Image
-
-from ipfml import processing, utils
-
-from modules.utils import config as cfg
-from modules.utils import data_type as dt
-from modules import noise
-
-import matplotlib.pyplot as plt
-
-noise_list            = cfg.noise_labels
-generated_folder      = cfg.generated_folder
-filename_ext          = cfg.filename_ext
-metric_choices        = cfg.metric_choices_labels
-normalization_choices = cfg.normalization_choices
-pictures_folder       = cfg.pictures_output_folder
-
-step_picture          = 10
-
-def main():
-
-    # default values
-    p_step = 1
-    p_color = 0
-    p_norm = 0
-    p_ylim = (0, 1)
-
-    max_value_svd = 0
-    min_value_svd = sys.maxsize
-
-    if len(sys.argv) <= 1:
-        print('python noise_svd_visualization.py --prefix generated/prefix/noise --metric lab --mode svdn --n 300 --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1"')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "h:p:m:m:n:i:s:c:n:y", ["help=", "prefix=", "metric=", "mode=", "n=", "interval=", "step=", "color=", "norm=", "ylim="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('python noise_svd_visualization.py --prefix generated/prefix/noise --metric lab --mode svdn --n 300 --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1"')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('python noise_svd_visualization.py --prefix generated/prefix/noise --metric lab --mode svdn --n 300 --interval "0, 200" --step 30 --color 1 --norm 1 --ylim "0, 1"')
-            sys.exit()
-        elif o in ("-p", "--prefix"):
-            p_path = a
-        elif o in ("-m", "--mode"):
-            p_mode = a
-
-            if not p_mode in normalization_choices:
-                assert False, "Unknown normalization choice, %s" % normalization_choices
-
-        elif o in ("-m", "--metric"):
-            p_metric = a
-
-            if not p_metric in metric_choices:
-                assert False, "Unknown metric choice, %s" % metric_choices
-
-        elif o in ("-n", "--n"):
-            p_n = int(a)
-        elif o in ("-n", "--norm"):
-            p_norm = int(a)
-        elif o in ("-c", "--color"):
-            p_color = int(a)
-        elif o in ("-i", "--interval"):
-            p_interval = list(map(int, a.split(',')))
-        elif o in ("-s", "--step"):
-            p_step = int(a)
-        elif o in ("-y", "--ylim"):
-            p_ylim = list(map(float, a.split(',')))
-        else:
-            assert False, "unhandled option"
-
-
-    p_prefix = p_path.split('/')[1].replace('_', '')
-    noise_name = p_path.split('/')[2]
-
-    if p_color:
-        file_path = p_path + "/" + p_prefix + "_" + noise_name + "_color_{}." + filename_ext
-    else:
-        file_path = p_path + "/" + p_prefix + "_" + noise_name + "_{}." + filename_ext
-
-    begin, end = p_interval
-    all_svd_data = []
-
-    svd_data = []
-    image_indices = []
-
-    # get all data from images
-    for i in range(1, p_n):
-
-        if i % step_picture == 0:
-
-            image_path = file_path.format(str(i))
-            img = Image.open(image_path)
-
-            svd_values = dt.get_svd_data(p_metric, img)
-
-            if p_norm:
-                svd_values = svd_values[begin:end]
-
-            all_svd_data.append(svd_values)
-
-            # update min max values
-            min_value = svd_values.min()
-            max_value = svd_values.max()
-
-            if min_value < min_value_svd:
-                min_value_svd = min_value
-
-            if max_value > max_value_svd:
-                max_value_svd = max_value
-
-            print('%.2f%%' % ((i + 1) / p_n * 100))
-            sys.stdout.write("\033[F")
-
-    for id, data in enumerate(all_svd_data):
-
-        if (id * step_picture) % p_step == 0:
-
-            current_data = data
-            if p_mode == 'svdn':
-                current_data = utils.normalize_arr(current_data)
-
-            if p_mode == 'svdne':
-                current_data = utils.normalize_arr_with_range(current_data, min_value_svd, max_value_svd)
-
-            svd_data.append(current_data)
-            image_indices.append(str(id * step_picture))
-
-    # display all data using matplotlib (configure plt)
-
-    plt.rcParams['figure.figsize'] = (25, 18)
-
-    plt.title(p_prefix  + ' noise, interval information ['+ str(begin) +', '+ str(end) +'], ' + p_metric + ' metric, step ' + str(p_step) + ' normalization ' + p_mode, fontsize=20)
-    plt.ylabel('Importance of noise [1, 999]', fontsize=14)
-    plt.xlabel('Vector features', fontsize=16)
-
-    for id, data in enumerate(svd_data):
-
-        p_label = p_prefix + str(image_indices[id])
-        plt.plot(data, label=p_label)
-
-    plt.legend(bbox_to_anchor=(0.8, 1), loc=2, borderaxespad=0.2, fontsize=14)
-
-    if not p_norm:
-        plt.xlim(begin, end)
-
-    # adapt ylim
-    y_begin, y_end = p_ylim
-    plt.ylim(y_begin, y_end)
-
-    output_filename = p_prefix + "_" + noise_name + "_1_to_" + str(p_n) + "_B" + str(begin) + "_E" + str(end) + "_" + p_metric + "_S" + str(p_step) + "_norm" + str(p_norm )+  "_" + p_mode
-
-    if p_color:
-        output_filename = output_filename + '_color'
-
-    print("Generation of output figure... %s" % output_filename)
-    output_path = os.path.join(pictures_folder, output_filename)
-
-    if not os.path.exists(pictures_folder):
-        os.makedirs(pictures_folder)
-
-    plt.savefig(output_path, dpi=(200))
-
-
-
-if __name__== "__main__":
-    main()