Browse Source

Prepare data using new dataset format (thresholds saved in file)

Jérôme BUISINE 3 months ago
parent
commit
abb0ca03ba

File diff suppressed because it is too large
+ 110 - 14
analysis/data_local_mean.ipynb


+ 1 - 1
cnn_models.py

@@ -119,7 +119,7 @@ def generate_model_3D(_input_shape, _weights_file=None):
     model.add(BatchNormalization())
     model.add(Dropout(0.5))
 
-    model.add(Dense(1))
+    model.add(Dense(2))
     model.add(Activation('sigmoid'))
 
     # reload weights if exists

+ 8 - 1
custom_config.py

@@ -1,5 +1,7 @@
 from modules.config.cnn_config import *
 
+import os
+
 # store all variables from cnn config
 context_vars = vars()
 
@@ -7,6 +9,11 @@ context_vars = vars()
 
 # folders
 
+output_data_folder              = 'data'
+output_data_generated           = os.path.join(output_data_folder, 'generated')
+output_datasets                 = os.path.join(output_data_folder, 'datasets')
+output_zones_learned            = os.path.join(output_data_folder, 'learned_zones')
+
 ## noisy_folder                    = 'noisy'
 ## not_noisy_folder                = 'notNoisy'
 backup_model_folder             = 'models_backup'
@@ -29,4 +36,4 @@ keras_epochs                    = 30
 ## keras_batch                     = 32
 ## val_dataset_size                = 0.2
 
-keras_img_size                  = (100, 100)
+keras_img_size                  = (200, 200)

+ 260 - 0
generate/generate_reconstructed_folder.py

@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jun 19 11:47:42 2019
+
+@author: jbuisine
+"""
+
+# main imports
+import sys, os, argparse
+import numpy as np
+
+# images processing imports
+from PIL import Image
+from ipfml.processing.segmentation import divide_in_blocks
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils.data import get_scene_image_quality
+from modules.classes.Transformation import Transformation
+
+# getting configuration information
+zone_folder             = cfg.zone_folder
+
+# define all scenes values
+zones                   = cfg.zones_indices
+features_choices        = cfg.features_choices_labels
+
+'''
+Display progress information as progress bar
+'''
+def write_progress(progress):
+    barWidth = 180
+
+    output_str = "["
+    pos = barWidth * progress
+    for i in range(barWidth):
+        if i < pos:
+           output_str = output_str + "="
+        elif i == pos:
+           output_str = output_str + ">"
+        else:
+            output_str = output_str + " "
+
+    output_str = output_str + "] " + str(int(progress * 100.0)) + " %\r"
+    print(output_str)
+    sys.stdout.write("\033[F")
+
+
+def generate_data(transformation, _dataset_path, _output, _human_thresholds, _replace):
+    """
+    @brief Method which generates all .csv files from scenes
+    @return nothing
+    """
+
+    # path is the default dataset path
+    scenes = os.listdir(_dataset_path)
+    n_scenes = len(scenes)
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(scenes):
+
+        print('Scene {0} of {1} ({2})'.format((id_scene + 1), n_scenes, folder_scene))
+        scene_path = os.path.join(_dataset_path, folder_scene)
+        output_scene_path = os.path.join(_output, folder_scene)
+
+        # construct each zones folder name
+        zones_folder = []
+        features_folder = []
+        zones_threshold = _human_thresholds[folder_scene]
+
+        # get zones list info
+        for index in zones:
+            index_str = str(index)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+
+            current_zone = "zone"+index_str
+            zones_folder.append(current_zone)
+            zone_path = os.path.join(output_scene_path, current_zone)
+
+            # custom path for feature
+            feature_path = os.path.join(zone_path, transformation.getName())
+
+            if not os.path.exists(feature_path):
+                os.makedirs(feature_path)
+
+            # custom path for interval of reconstruction and feature
+            feature_interval_path = os.path.join(zone_path, transformation.getTransformationPath())
+            features_folder.append(feature_interval_path)
+
+            if not os.path.exists(feature_interval_path):
+                os.makedirs(feature_interval_path)
+
+            # create for each zone the labels folder
+            labels = [cfg.not_noisy_folder, cfg.noisy_folder]
+
+            for label in labels:
+                label_folder = os.path.join(feature_interval_path, label)
+
+                if not os.path.exists(label_folder):
+                    os.makedirs(label_folder)
+
+        # get all images of folder
+        scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
+        number_scene_image = len(scene_images)
+
+        # for each images
+        for id_img, img_path in enumerate(scene_images):
+
+            current_img = Image.open(img_path)
+            img_blocks = divide_in_blocks(current_img, cfg.sub_image_size)
+
+            current_quality_index = int(get_scene_image_quality(img_path))
+
+            for id_block, block in enumerate(img_blocks):
+
+                ##########################
+                # Image computation part #
+                ##########################
+
+                label_path = features_folder[id_block]
+
+                # get label folder for block
+                if current_quality_index > zones_threshold[id_block]:
+                    label_path = os.path.join(label_path, cfg.not_noisy_folder)
+                else:
+                    label_path = os.path.join(label_path, cfg.noisy_folder)
+
+                # check if necessary to compute or not images
+                # Data augmentation!
+                rotations = [0, 90, 180, 270]
+
+                #img_flip_labels = ['original', 'horizontal', 'vertical', 'both']
+                img_flip_labels = ['original', 'horizontal']
+
+                output_images_path = []
+                check_path_exists = []
+                # rotate and flip image to increase dataset size
+                for id, flip_label in enumerate(img_flip_labels):
+                    for rotation in rotations:
+                        output_reconstructed_filename = img_path.split('/')[-1].replace('.png', '') + '_' + zones_folder[id_block] + cfg.post_image_name_separator
+                        output_reconstructed_filename = output_reconstructed_filename + flip_label + '_' + str(rotation) + '.png'
+                        output_reconstructed_path = os.path.join(label_path, output_reconstructed_filename)
+
+                        if os.path.exists(output_reconstructed_path):
+                            check_path_exists.append(True)
+                        else:
+                            check_path_exists.append(False)
+
+                        output_images_path.append(output_reconstructed_path)
+
+                # compute only if not exists or necessary to replace
+                if _replace or not np.array(check_path_exists).all():
+                    # compute image
+                    # pass block to grey level
+                    output_block = transformation.getTransformedImage(block)
+                    output_block = np.array(output_block, 'uint8')
+                    
+                    # current output image
+                    output_block_img = Image.fromarray(output_block)
+
+                    #horizontal_img = output_block_img.transpose(Image.FLIP_LEFT_RIGHT)
+                    #vertical_img = output_block_img.transpose(Image.FLIP_TOP_BOTTOM)
+                    #both_img = output_block_img.transpose(Image.TRANSPOSE)
+
+                    #flip_images = [output_block_img, horizontal_img, vertical_img, both_img]
+                    #flip_images = [output_block_img, horizontal_img]
+
+                    # Only current image img currenlty
+                    flip_images = [output_block_img]
+
+                    # rotate and flip image to increase dataset size
+                    counter_index = 0 # get current path index
+                    for id, flip in enumerate(flip_images):
+                        for rotation in rotations:
+
+                            if _replace or not check_path_exists[counter_index]:
+                                rotated_output_img = flip.rotate(rotation)
+                                rotated_output_img.save(output_images_path[counter_index])
+
+                            counter_index +=1
+
+            write_progress((id_img + 1) / number_scene_image)
+
+        print('\n')
+
+    print("{0}_{1} : end of data generation\n".format(transformation.getName(), transformation.getParam()))
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes using specific interval if necessary")
+
+    parser.add_argument('--features', type=str, 
+                                     help="list of features choice in order to compute data",
+                                     default='svd_reconstruction, ipca_reconstruction',
+                                     required=True)
+    parser.add_argument('--params', type=str, 
+                                    help="list of specific param for each feature choice (See README.md for further information in 3D mode)", 
+                                    default='100, 200 :: 50, 25',
+                                    required=True)
+    parser.add_argument('--folder', type=str,
+                            help='folder where dataset is available',
+                            required=True)  
+    parser.add_argument('--output', type=str,
+                                help='output folder where data are saved',
+                                required=True)              
+    parser.add_argument('--thresholds', type=str, help='file which cantains all thresholds', required=True)                  
+    parser.add_argument('--size', type=str, 
+                                help="specific size of image", 
+                                default='100, 100',
+                                required=True)
+    parser.add_argument('--replace', type=int, help='replace previous picutre', default=1)
+
+    args = parser.parse_args()
+
+    p_features   = list(map(str.strip, args.features.split(',')))
+    p_params     = list(map(str.strip, args.params.split('::')))
+    p_folder     = args.folder
+    p_output     = args.output
+    p_thresholds = args.thresholds
+    p_size       = args.size
+    p_replace    = bool(args.replace)
+
+    # list of transformations
+    transformations = []
+
+    for id, feature in enumerate(p_features):
+
+        if feature not in features_choices or feature == 'static':
+            raise ValueError("Unknown feature, please select a correct feature (`static` excluded) : ", features_choices)
+
+        transformations.append(Transformation(feature, p_params[id], p_size))
+
+    human_thresholds = {}
+
+    # 3. retrieve human_thresholds
+    # construct zones folder
+    with open(p_thresholds) as f:
+        thresholds_line = f.readlines()
+
+        for line in thresholds_line:
+            data = line.split(';')
+            del data[-1] # remove unused last element `\n`
+            current_scene = data[0]
+            thresholds_scene = data[1:]
+
+            if current_scene != '50_shades_of_grey':
+                human_thresholds[current_scene] = [ int(threshold) for threshold in  thresholds_scene ]
+
+
+    # generate all or specific feature data
+    for transformation in transformations:
+        generate_data(transformation, p_folder, p_output, human_thresholds, p_replace)
+
+if __name__== "__main__":
+    main()

+ 104 - 0
generate/generate_selected_zones_file.py

@@ -0,0 +1,104 @@
+# main imports
+import numpy as np
+import pandas as pd
+import sys, os, argparse
+import random
+
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+
+
+def save_learned_zones(output_name, scene, zones):
+
+    if not os.path.exists(cfg.output_zones_learned):
+        os.makedirs(cfg.output_zones_learned)
+
+    with open(os.path.join(cfg.output_zones_learned, output_name), 'a') as f:
+        f.write(scene + ';')
+
+        for zone in zones:
+            f.write(str(zone) + ';')
+
+        f.write('\n')
+
+
+def get_random_zones(scene, zones, n_zones):
+
+    random.shuffle(zones)
+
+    # specific case for 'Cuisine01' (zone 12 is also noisy even in reference image)
+    # if scene == 'Cuisine01':
+    #     while 12 in zones[0:n_zones]:
+    #         random.shuffle(zones)
+    
+    return zones[0:n_zones]
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Read and compute entropy data file (using diff)")
+
+    parser.add_argument('--folder', type=str, help='dataset scene folder', required=True)
+    parser.add_argument('--n_zones', type=int, help='number of zones used in train', default=10)
+    parser.add_argument('--output', type=str, help='file with specific training zone', required=True)
+    parser.add_argument('--thresholds', type=str, help='file with specific thresholds (using only scene from this file', default='')
+
+    args = parser.parse_args()
+
+    p_folder       = args.folder
+    p_n_zones      = args.n_zones
+    p_output       = args.output
+    p_thresholds   = args.thresholds
+
+    # extract scenes to use if specified
+    available_scenes = None
+
+    if len(p_thresholds) > 0:
+        
+        available_scenes = []
+
+        with open(p_thresholds) as f:
+            thresholds_line = f.readlines()
+
+            for line in thresholds_line:
+                data = line.split(';')
+                del data[-1] # remove unused last element `\n`
+                current_scene = data[0]
+
+                # need to rename `current_name` because we only used part6
+                # scene_split = current_scene.split('_')
+                # del scene_split[-1]
+                # scene_name = '_'.join(scene_split)
+
+                available_scenes.append(current_scene)
+
+
+    # specific number of zones (zones indices)
+    zones = np.arange(16)
+
+    # get all scene names
+    scenes = os.listdir(p_folder)
+
+    # create output thresholds directory if necessary
+    folder, _ = os.path.split(p_output)
+
+    if len(folder) > 0:
+        os.makedirs(folder)
+
+    # for each scene we generate random zones choice
+    for folder_scene in scenes:
+
+        if available_scenes is not None:
+
+            if folder_scene in available_scenes:
+                selected_zones = get_random_zones(folder_scene, zones, p_n_zones)
+                save_learned_zones(p_output, folder_scene, selected_zones)
+        else:
+            selected_zones = get_random_zones(folder_scene, zones, p_n_zones)
+            save_learned_zones(p_output, folder_scene, selected_zones)
+            
+
+if __name__== "__main__":
+    main()

+ 1 - 1
modules

@@ -1 +1 @@
-Subproject commit acdc213cdd418eb03833f7f0f0a72466b8812cbf
+Subproject commit 270de3a969ff3121e68f435cc6a3b570ba5b9d69

+ 45 - 21
run/cross_run_nl_mean.sh

@@ -6,41 +6,65 @@ all_scenes="A,B,C,D,E,F,G,H,I"
 # file which contains model names we want to use for simulation
 file_path="results/models_comparisons.csv"
 stride=1
-dist_patch=6
+window=6
 
 # for kernel in {3,5,7}; do
 #     echo python generate/generate_reconstructed_data.py --features ${metric} --params ${kernel},${dist_patch} --size 100,100 --scenes ${all_scenes} --replace 0
 # done
 
-for scene in {"A","B","D","G","H","I"}; do
+# for scene in {"A","B","D","G","H","I"}; do
 
-    # remove current scene test from dataset
-    s="${scenes//,${scene}}"
-    s="${s//${scene},}"
+#     # remove current scene test from dataset
+#     s="${scenes//,${scene}}"
+#     s="${s//${scene},}"
 
-    for zone in {10,11,12}; do
-        for kernel in {3,5,7}; do
-            for balancing in {0,1}; do
+#     for zone in {10,11,12}; do
+#         for kernel in {3,5,7}; do
+#             for balancing in {0,1}; do
             
-                OUTPUT_DATA_FILE="${metric}_nb_zones_${zone}_W${window}_K${kernel}_balancing${balancing}_without_${scene}"
-                OUTPUT_DATA_FILE_TEST="${metric}_nb_zones_${zone}_W${window}_K${kernel}_balancing${balancing}_scene_${scene}"
+#                 OUTPUT_DATA_FILE="${metric}_nb_zones_${zone}_W${window}_K${kernel}_balancing${balancing}_without_${scene}"
+#                 OUTPUT_DATA_FILE_TEST="${metric}_nb_zones_${zone}_W${window}_K${kernel}_balancing${balancing}_scene_${scene}"
 
-                if grep -q "${OUTPUT_DATA_FILE}" "${file_path}"; then
+#                 if grep -q "${OUTPUT_DATA_FILE}" "${file_path}"; then
                 
-                    echo "SVD model ${OUTPUT_DATA_FILE} already generated"
+#                     echo "SVD model ${OUTPUT_DATA_FILE} already generated"
 
-                else
+#                 else
 
-                    #echo "Run computation for SVD model ${OUTPUT_DATA_FILE}"
-                    echo python generate/generate_dataset.py --output data/${OUTPUT_DATA_FILE_TEST} --features ${metric} --scenes ${scene} --params ${kernel},${dist_patch} --nb_zones ${zone} --random 1 --size 200,200     
+#                     #echo "Run computation for SVD model ${OUTPUT_DATA_FILE}"
+#                     echo python generate/generate_dataset.py --output data/${OUTPUT_DATA_FILE_TEST} --features ${metric} --scenes ${scene} --params ${kernel},${dist_patch} --nb_zones ${zone} --random 1 --size 200,200     
 
-                    echo python generate/generate_dataset.py --output data/${OUTPUT_DATA_FILE} --features ${metric} --scenes ${s} --params ${kernel},${dist_patch} --nb_zones ${zone} --random 1 --size 200,200     
+#                     echo python generate/generate_dataset.py --output data/${OUTPUT_DATA_FILE} --features ${metric} --scenes ${s} --params ${kernel},${dist_patch} --nb_zones ${zone} --random 1 --size 200,200     
                     
-                    echo python train_model.py --data data/${OUTPUT_DATA_FILE} --output ${OUTPUT_DATA_FILE} --balancing ${balancing}
-                    echo python prediction_model.py --data data/${OUTPUT_DATA_FILE_TEST}.train --model saved_models/${OUTPUT_DATA_FILE}.json
-                fi 
-            done
+#                     echo python train_model.py --data data/${OUTPUT_DATA_FILE} --output ${OUTPUT_DATA_FILE} --balancing ${balancing} --chanels 3
+#                     echo python prediction_model.py --data data/${OUTPUT_DATA_FILE_TEST}.train --model saved_models/${OUTPUT_DATA_FILE}.json
+#                 fi 
+#             done
+#         done
+#     done
+# done
+
+
+s="A,D,G,H"
+for zone in {10,11,12}; do
+    for kernel in {3,5,7}; do
+        for balancing in {0,1}; do
+        
+            OUTPUT_DATA_FILE="${metric}_nb_zones_${zone}_W${window}_K${kernel}_balancing${balancing}_maxwell"
+
+            if grep -q "${OUTPUT_DATA_FILE}" "${file_path}"; then
+            
+                echo "SVD model ${OUTPUT_DATA_FILE} already generated"
+
+            else
+
+                #echo "Run computation for SVD model ${OUTPUT_DATA_FILE}"
+
+                echo python generate/generate_dataset.py --output data/${OUTPUT_DATA_FILE} --features ${metric} --scenes ${s} --params ${kernel},${dist_patch} --nb_zones ${zone} --random 1 --size 200,200     
+                
+                echo python train_model.py --data data/${OUTPUT_DATA_FILE} --output ${OUTPUT_DATA_FILE} --balancing ${balancing} --chanels 3
+                echo python prediction_model.py --data data/${OUTPUT_DATA_FILE_TEST}.test --model saved_models/${OUTPUT_DATA_FILE}.json
+            fi 
         done
     done
 done
-