generate_dataset_sequence_file.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Wed Jun 19 11:47:42 2019
  5. @author: jbuisine
  6. """
  7. # main imports
  8. import sys, os, argparse
  9. import numpy as np
  10. import random
  11. # images processing imports
  12. from PIL import Image
  13. from ipfml.processing.segmentation import divide_in_blocks
  14. # modules imports
  15. sys.path.insert(0, '') # trick to enable import of main folder module
  16. import custom_config as cfg
  17. from modules.utils import data as dt
  18. from modules.classes.Transformation import Transformation
  19. # getting configuration information
  20. zone_folder = cfg.zone_folder
  21. learned_folder = cfg.learned_zones_folder
  22. min_max_filename = cfg.min_max_filename_extension
  23. # define all scenes values
  24. scenes_list = cfg.scenes_names
  25. scenes_indices = cfg.scenes_indices
  26. dataset_path = cfg.dataset_path
  27. zones = cfg.zones_indices
  28. seuil_expe_filename = cfg.seuil_expe_filename
  29. features_choices = cfg.features_choices_labels
  30. output_data_folder = cfg.output_datasets
  31. generic_output_file_svd = '_random.csv'
  32. def generate_data_model(_filename, _transformations, _dataset_folder, _selected_zones, _sequence):
  33. output_train_filename = os.path.join(output_data_folder, _filename, _filename + ".train")
  34. output_test_filename = os.path.join(output_data_folder, _filename, _filename + ".val")
  35. # create path if not exists
  36. if not os.path.exists(os.path.join(output_data_folder, _filename)):
  37. os.makedirs(os.path.join(output_data_folder, _filename))
  38. train_file = open(output_train_filename, 'w')
  39. test_file = open(output_test_filename, 'w')
  40. # train_file_data = []
  41. # test_file_data = []
  42. # specific number of zones (zones indices)
  43. zones = np.arange(16)
  44. # go ahead each scenes
  45. for folder_scene in _selected_zones:
  46. scene_path = os.path.join(_dataset_folder, folder_scene)
  47. train_zones = _selected_zones[folder_scene]
  48. for id_zone, index_folder in enumerate(zones):
  49. index_str = str(index_folder)
  50. if len(index_str) < 2:
  51. index_str = "0" + index_str
  52. current_zone_folder = "zone" + index_str
  53. zone_path = os.path.join(scene_path, current_zone_folder)
  54. # custom path for interval of reconstruction and metric
  55. features_path = []
  56. for transformation in _transformations:
  57. # check if it's a static content and create augmented images if necessary
  58. if transformation.getName() == 'static':
  59. # {sceneName}/zoneXX/static
  60. static_metric_path = os.path.join(zone_path, transformation.getName())
  61. # img.png
  62. image_name = transformation.getParam().split('/')[-1]
  63. # {sceneName}/zoneXX/static/img
  64. image_prefix_name = image_name.replace('.png', '')
  65. image_folder_path = os.path.join(static_metric_path, image_prefix_name)
  66. if not os.path.exists(image_folder_path):
  67. os.makedirs(image_folder_path)
  68. features_path.append(image_folder_path)
  69. # get image path to manage
  70. # {sceneName}/static/img.png
  71. transform_image_path = os.path.join(scene_path, transformation.getName(), image_name)
  72. static_transform_image = Image.open(transform_image_path)
  73. static_transform_image_block = divide_in_blocks(static_transform_image, cfg.sub_image_size)[id_zone]
  74. dt.augmented_data_image(static_transform_image_block, image_folder_path, image_prefix_name)
  75. else:
  76. metric_interval_path = os.path.join(zone_path, transformation.getTransformationPath())
  77. features_path.append(metric_interval_path)
  78. # as labels are same for each metric
  79. for label in os.listdir(features_path[0]):
  80. label_features_path = []
  81. for path in features_path:
  82. label_path = os.path.join(path, label)
  83. label_features_path.append(label_path)
  84. # getting images list for each metric
  85. features_images_list = []
  86. for index_metric, label_path in enumerate(label_features_path):
  87. if _transformations[index_metric].getName() == 'static':
  88. # by default append nothing..
  89. features_images_list.append([])
  90. else:
  91. images = sorted(os.listdir(label_path))
  92. features_images_list.append(images)
  93. sequence_data = []
  94. # construct each line using all images path of each
  95. for index_image in range(0, len(features_images_list[0])):
  96. images_path = []
  97. # get information about rotation and flip from first transformation (need to be a not static transformation)
  98. current_post_fix = features_images_list[0][index_image].split(cfg.post_image_name_separator)[-1]
  99. # getting images with same index and hence name for each metric (transformation)
  100. for index_metric in range(0, len(features_path)):
  101. # custom behavior for static transformation (need to check specific image)
  102. if _transformations[index_metric].getName() == 'static':
  103. # add static path with selecting correct data augmented image
  104. image_name = _transformations[index_metric].getParam().split('/')[-1].replace('.png', '')
  105. img_path = os.path.join(features_path[index_metric], image_name + cfg.post_image_name_separator + current_post_fix)
  106. images_path.append(img_path)
  107. else:
  108. img_path = features_images_list[index_metric][index_image]
  109. images_path.append(os.path.join(label_features_path[index_metric], img_path))
  110. if label == cfg.noisy_folder:
  111. line = '1;'
  112. else:
  113. line = '0;'
  114. # add new data information into sequence
  115. sequence_data.append(images_path)
  116. if len(sequence_data) >= _sequence:
  117. # prepare whole line for LSTM model kind
  118. # keeping last noisy label
  119. for id_seq, seq_images_path in enumerate(sequence_data):
  120. # compute line information with all images paths
  121. for id_path, img_path in enumerate(seq_images_path):
  122. if id_path < len(seq_images_path) - 1:
  123. line = line + img_path + '::'
  124. else:
  125. line = line + img_path
  126. if id_seq < len(sequence_data) - 1:
  127. line += ';'
  128. line = line + '\n'
  129. if id_zone in train_zones:
  130. # train_file_data.append(line)
  131. train_file.write(line)
  132. else:
  133. # test_file_data.append(line)
  134. test_file.write(line)
  135. # remove first element (sliding window)
  136. del sequence_data[0]
  137. # random.shuffle(train_file_data)
  138. # random.shuffle(test_file_data)
  139. # for line in train_file_data:
  140. # train_file.write(line)
  141. # for line in test_file_data:
  142. # test_file.write(line)
  143. train_file.close()
  144. test_file.close()
  145. def main():
  146. parser = argparse.ArgumentParser(description="Compute specific dataset for model using of metric")
  147. parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
  148. parser.add_argument('--folder', type=str,
  149. help='folder where generated data are available',
  150. required=True)
  151. parser.add_argument('--features', type=str,
  152. help="list of features choice in order to compute data",
  153. default='svd_reconstruction, ipca_reconstruction',
  154. required=True)
  155. parser.add_argument('--params', type=str,
  156. help="list of specific param for each metric choice (See README.md for further information in 3D mode)",
  157. default='100, 200 :: 50, 25',
  158. required=True)
  159. parser.add_argument('--sequence', type=int, help='sequence length expected', required=True)
  160. parser.add_argument('--size', type=str,
  161. help="Size of input images",
  162. default="100, 100")
  163. parser.add_argument('--selected_zones', type=str, help='file which contains all selected zones of scene', required=True)
  164. args = parser.parse_args()
  165. p_filename = args.output
  166. p_folder = args.folder
  167. p_features = list(map(str.strip, args.features.split(',')))
  168. p_params = list(map(str.strip, args.params.split('::')))
  169. p_sequence = args.sequence
  170. p_size = args.size # not necessary to split here
  171. p_selected_zones = args.selected_zones
  172. selected_zones = {}
  173. with(open(p_selected_zones, 'r')) as f:
  174. for line in f.readlines():
  175. data = line.split(';')
  176. del data[-1]
  177. scene_name = data[0]
  178. thresholds = data[1:]
  179. selected_zones[scene_name] = [ int(t) for t in thresholds ]
  180. # create list of Transformation
  181. transformations = []
  182. for id, feature in enumerate(p_features):
  183. if feature not in features_choices:
  184. raise ValueError("Unknown metric, please select a correct metric : ", features_choices)
  185. transformations.append(Transformation(feature, p_params[id], p_size))
  186. if transformations[0].getName() == 'static':
  187. raise ValueError("The first transformation in list cannot be static")
  188. # create database using img folder (generate first time only)
  189. generate_data_model(p_filename, transformations, p_folder, selected_zones, p_sequence)
  190. if __name__== "__main__":
  191. main()