|
@@ -0,0 +1,260 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
+"""
|
|
|
+Created on Wed Jun 19 11:47:42 2019
|
|
|
+
|
|
|
+@author: jbuisine
|
|
|
+"""
|
|
|
+
|
|
|
+# main imports
|
|
|
+import sys, os, argparse
|
|
|
+import numpy as np
|
|
|
+
|
|
|
+# images processing imports
|
|
|
+from PIL import Image
|
|
|
+from ipfml.processing.segmentation import divide_in_blocks
|
|
|
+
|
|
|
+# modules imports
|
|
|
+sys.path.insert(0, '') # trick to enable import of main folder module
|
|
|
+
|
|
|
+import custom_config as cfg
|
|
|
+from modules.utils.data import get_scene_image_quality
|
|
|
+from modules.classes.Transformation import Transformation
|
|
|
+
|
|
|
+# getting configuration information
|
|
|
+zone_folder = cfg.zone_folder
|
|
|
+
|
|
|
+# define all scenes values
|
|
|
+zones = cfg.zones_indices
|
|
|
+features_choices = cfg.features_choices_labels
|
|
|
+
|
|
|
+'''
|
|
|
+Display progress information as progress bar
|
|
|
+'''
|
|
|
+def write_progress(progress):
|
|
|
+ barWidth = 180
|
|
|
+
|
|
|
+ output_str = "["
|
|
|
+ pos = barWidth * progress
|
|
|
+ for i in range(barWidth):
|
|
|
+ if i < pos:
|
|
|
+ output_str = output_str + "="
|
|
|
+ elif i == pos:
|
|
|
+ output_str = output_str + ">"
|
|
|
+ else:
|
|
|
+ output_str = output_str + " "
|
|
|
+
|
|
|
+ output_str = output_str + "] " + str(int(progress * 100.0)) + " %\r"
|
|
|
+ print(output_str)
|
|
|
+ sys.stdout.write("\033[F")
|
|
|
+
|
|
|
+
|
|
|
+def generate_data(transformation, _dataset_path, _output, _human_thresholds, _replace):
|
|
|
+ """
|
|
|
+ @brief Method which generates all .csv files from scenes
|
|
|
+ @return nothing
|
|
|
+ """
|
|
|
+
|
|
|
+ # path is the default dataset path
|
|
|
+ scenes = os.listdir(_dataset_path)
|
|
|
+ n_scenes = len(scenes)
|
|
|
+
|
|
|
+ # go ahead each scenes
|
|
|
+ for id_scene, folder_scene in enumerate(scenes):
|
|
|
+
|
|
|
+ print('Scene {0} of {1} ({2})'.format((id_scene + 1), n_scenes, folder_scene))
|
|
|
+ scene_path = os.path.join(_dataset_path, folder_scene)
|
|
|
+ output_scene_path = os.path.join(_output, folder_scene)
|
|
|
+
|
|
|
+ # construct each zones folder name
|
|
|
+ zones_folder = []
|
|
|
+ features_folder = []
|
|
|
+ zones_threshold = _human_thresholds[folder_scene]
|
|
|
+
|
|
|
+ # get zones list info
|
|
|
+ for index in zones:
|
|
|
+ index_str = str(index)
|
|
|
+ if len(index_str) < 2:
|
|
|
+ index_str = "0" + index_str
|
|
|
+
|
|
|
+ current_zone = "zone"+index_str
|
|
|
+ zones_folder.append(current_zone)
|
|
|
+ zone_path = os.path.join(output_scene_path, current_zone)
|
|
|
+
|
|
|
+ # custom path for feature
|
|
|
+ feature_path = os.path.join(zone_path, transformation.getName())
|
|
|
+
|
|
|
+ if not os.path.exists(feature_path):
|
|
|
+ os.makedirs(feature_path)
|
|
|
+
|
|
|
+ # custom path for interval of reconstruction and feature
|
|
|
+ feature_interval_path = os.path.join(zone_path, transformation.getTransformationPath())
|
|
|
+ features_folder.append(feature_interval_path)
|
|
|
+
|
|
|
+ if not os.path.exists(feature_interval_path):
|
|
|
+ os.makedirs(feature_interval_path)
|
|
|
+
|
|
|
+ # create for each zone the labels folder
|
|
|
+ labels = [cfg.not_noisy_folder, cfg.noisy_folder]
|
|
|
+
|
|
|
+ for label in labels:
|
|
|
+ label_folder = os.path.join(feature_interval_path, label)
|
|
|
+
|
|
|
+ if not os.path.exists(label_folder):
|
|
|
+ os.makedirs(label_folder)
|
|
|
+
|
|
|
+ # get all images of folder
|
|
|
+ scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
|
|
|
+ number_scene_image = len(scene_images)
|
|
|
+
|
|
|
+ # for each images
|
|
|
+ for id_img, img_path in enumerate(scene_images):
|
|
|
+
|
|
|
+ current_img = Image.open(img_path)
|
|
|
+ img_blocks = divide_in_blocks(current_img, cfg.sub_image_size)
|
|
|
+
|
|
|
+ current_quality_index = int(get_scene_image_quality(img_path))
|
|
|
+
|
|
|
+ for id_block, block in enumerate(img_blocks):
|
|
|
+
|
|
|
+ ##########################
|
|
|
+ # Image computation part #
|
|
|
+ ##########################
|
|
|
+
|
|
|
+ label_path = features_folder[id_block]
|
|
|
+
|
|
|
+ # get label folder for block
|
|
|
+ if current_quality_index > zones_threshold[id_block]:
|
|
|
+ label_path = os.path.join(label_path, cfg.not_noisy_folder)
|
|
|
+ else:
|
|
|
+ label_path = os.path.join(label_path, cfg.noisy_folder)
|
|
|
+
|
|
|
+ # check if necessary to compute or not images
|
|
|
+ # Data augmentation!
|
|
|
+ rotations = [0, 90, 180, 270]
|
|
|
+
|
|
|
+ #img_flip_labels = ['original', 'horizontal', 'vertical', 'both']
|
|
|
+ img_flip_labels = ['original', 'horizontal']
|
|
|
+
|
|
|
+ output_images_path = []
|
|
|
+ check_path_exists = []
|
|
|
+ # rotate and flip image to increase dataset size
|
|
|
+ for id, flip_label in enumerate(img_flip_labels):
|
|
|
+ for rotation in rotations:
|
|
|
+ output_reconstructed_filename = img_path.split('/')[-1].replace('.png', '') + '_' + zones_folder[id_block] + cfg.post_image_name_separator
|
|
|
+ output_reconstructed_filename = output_reconstructed_filename + flip_label + '_' + str(rotation) + '.png'
|
|
|
+ output_reconstructed_path = os.path.join(label_path, output_reconstructed_filename)
|
|
|
+
|
|
|
+ if os.path.exists(output_reconstructed_path):
|
|
|
+ check_path_exists.append(True)
|
|
|
+ else:
|
|
|
+ check_path_exists.append(False)
|
|
|
+
|
|
|
+ output_images_path.append(output_reconstructed_path)
|
|
|
+
|
|
|
+ # compute only if not exists or necessary to replace
|
|
|
+ if _replace or not np.array(check_path_exists).all():
|
|
|
+ # compute image
|
|
|
+ # pass block to grey level
|
|
|
+ output_block = transformation.getTransformedImage(block)
|
|
|
+ output_block = np.array(output_block, 'uint8')
|
|
|
+
|
|
|
+ # current output image
|
|
|
+ output_block_img = Image.fromarray(output_block)
|
|
|
+
|
|
|
+ #horizontal_img = output_block_img.transpose(Image.FLIP_LEFT_RIGHT)
|
|
|
+ #vertical_img = output_block_img.transpose(Image.FLIP_TOP_BOTTOM)
|
|
|
+ #both_img = output_block_img.transpose(Image.TRANSPOSE)
|
|
|
+
|
|
|
+ #flip_images = [output_block_img, horizontal_img, vertical_img, both_img]
|
|
|
+ #flip_images = [output_block_img, horizontal_img]
|
|
|
+
|
|
|
+ # Only current image img currenlty
|
|
|
+ flip_images = [output_block_img]
|
|
|
+
|
|
|
+ # rotate and flip image to increase dataset size
|
|
|
+ counter_index = 0 # get current path index
|
|
|
+ for id, flip in enumerate(flip_images):
|
|
|
+ for rotation in rotations:
|
|
|
+
|
|
|
+ if _replace or not check_path_exists[counter_index]:
|
|
|
+ rotated_output_img = flip.rotate(rotation)
|
|
|
+ rotated_output_img.save(output_images_path[counter_index])
|
|
|
+
|
|
|
+ counter_index +=1
|
|
|
+
|
|
|
+ write_progress((id_img + 1) / number_scene_image)
|
|
|
+
|
|
|
+ print('\n')
|
|
|
+
|
|
|
+ print("{0}_{1} : end of data generation\n".format(transformation.getName(), transformation.getParam()))
|
|
|
+
|
|
|
+
|
|
|
+def main():
|
|
|
+
|
|
|
+ parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes using specific interval if necessary")
|
|
|
+
|
|
|
+ parser.add_argument('--features', type=str,
|
|
|
+ help="list of features choice in order to compute data",
|
|
|
+ default='svd_reconstruction, ipca_reconstruction',
|
|
|
+ required=True)
|
|
|
+ parser.add_argument('--params', type=str,
|
|
|
+ help="list of specific param for each feature choice (See README.md for further information in 3D mode)",
|
|
|
+ default='100, 200 :: 50, 25',
|
|
|
+ required=True)
|
|
|
+ parser.add_argument('--folder', type=str,
|
|
|
+ help='folder where dataset is available',
|
|
|
+ required=True)
|
|
|
+ parser.add_argument('--output', type=str,
|
|
|
+ help='output folder where data are saved',
|
|
|
+ required=True)
|
|
|
+ parser.add_argument('--thresholds', type=str, help='file which cantains all thresholds', required=True)
|
|
|
+ parser.add_argument('--size', type=str,
|
|
|
+ help="specific size of image",
|
|
|
+ default='100, 100',
|
|
|
+ required=True)
|
|
|
+ parser.add_argument('--replace', type=int, help='replace previous picutre', default=1)
|
|
|
+
|
|
|
+ args = parser.parse_args()
|
|
|
+
|
|
|
+ p_features = list(map(str.strip, args.features.split(',')))
|
|
|
+ p_params = list(map(str.strip, args.params.split('::')))
|
|
|
+ p_folder = args.folder
|
|
|
+ p_output = args.output
|
|
|
+ p_thresholds = args.thresholds
|
|
|
+ p_size = args.size
|
|
|
+ p_replace = bool(args.replace)
|
|
|
+
|
|
|
+ # list of transformations
|
|
|
+ transformations = []
|
|
|
+
|
|
|
+ for id, feature in enumerate(p_features):
|
|
|
+
|
|
|
+ if feature not in features_choices or feature == 'static':
|
|
|
+ raise ValueError("Unknown feature, please select a correct feature (`static` excluded) : ", features_choices)
|
|
|
+
|
|
|
+ transformations.append(Transformation(feature, p_params[id], p_size))
|
|
|
+
|
|
|
+ human_thresholds = {}
|
|
|
+
|
|
|
+ # 3. retrieve human_thresholds
|
|
|
+ # construct zones folder
|
|
|
+ with open(p_thresholds) as f:
|
|
|
+ thresholds_line = f.readlines()
|
|
|
+
|
|
|
+ for line in thresholds_line:
|
|
|
+ data = line.split(';')
|
|
|
+ del data[-1] # remove unused last element `\n`
|
|
|
+ current_scene = data[0]
|
|
|
+ thresholds_scene = data[1:]
|
|
|
+
|
|
|
+ if current_scene != '50_shades_of_grey':
|
|
|
+ human_thresholds[current_scene] = [ int(threshold) for threshold in thresholds_scene ]
|
|
|
+
|
|
|
+
|
|
|
+ # generate all or specific feature data
|
|
|
+ for transformation in transformations:
|
|
|
+ generate_data(transformation, p_folder, p_output, human_thresholds, p_replace)
|
|
|
+
|
|
|
+if __name__== "__main__":
|
|
|
+ main()
|