generate_all_data.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Fri Sep 14 21:02:42 2018
  5. @author: jbuisine
  6. """
  7. from __future__ import print_function
  8. import sys, os, getopt
  9. import numpy as np
  10. import random
  11. import time
  12. import json
  13. from modules.utils.data_type import get_svd_data
  14. from PIL import Image
  15. from ipfml import processing
  16. from ipfml import metrics
  17. from skimage import color
  18. from modules.utils import config as cfg
  19. # getting configuration information
  20. zone_folder = cfg.zone_folder
  21. min_max_filename = cfg.min_max_filename_extension
  22. # define all scenes values
  23. scenes_list = cfg.maxwell_scenes_folders
  24. scenes_indexes = cfg.scenes_indices
  25. choices = cfg.normalization_choices
  26. path = cfg.generated_folder
  27. zones = cfg.zones_indices
  28. seuil_expe_filename = cfg.seuil_expe_filename
  29. noise_choices = cfg.noise_labels
  30. metric_choices = cfg.metric_choices_labels
  31. output_data_folder = cfg.output_data_folder
  32. end_counter_index = cfg.default_number_of_images
  33. generic_output_file_svd = '_random.csv'
  34. picture_step = 10
  35. # avoid calibration data ?
  36. calibration_folder = 'calibration'
  37. def generate_data_svd(data_type, color, mode):
  38. """
  39. @brief Method which generates all .csv files from scenes
  40. @param data_type, metric choice
  41. @param mode, normalization choice
  42. @return nothing
  43. """
  44. scenes = os.listdir(path)
  45. # filter scene
  46. scenes = [s for s in scenes if calibration_folder not in s]
  47. # remove min max file from scenes folder
  48. scenes = [s for s in scenes if min_max_filename not in s]
  49. # keep in memory min and max data found from data_type
  50. min_val_found = sys.maxsize
  51. max_val_found = 0
  52. data_min_max_filename = os.path.join(path, data_type + min_max_filename)
  53. # go ahead each scenes
  54. for id_scene, folder_scene in enumerate(scenes):
  55. print(folder_scene)
  56. scene_path = os.path.join(path, folder_scene)
  57. for noise in noise_choices:
  58. noise_path = os.path.join(scene_path, noise)
  59. # getting output filename
  60. if color:
  61. output_svd_filename = data_type + "_color_" + mode + generic_output_file_svd
  62. else:
  63. output_svd_filename = data_type + "_" + mode + generic_output_file_svd
  64. # construct each zones folder name
  65. zones_folder = []
  66. svd_output_files = []
  67. # get zones list info
  68. for index in zones:
  69. index_str = str(index)
  70. if len(index_str) < 2:
  71. index_str = "0" + index_str
  72. current_zone = "zone"+index_str
  73. zones_folder.append(current_zone)
  74. zone_path = os.path.join(noise_path, current_zone)
  75. if not os.path.exists(zone_path):
  76. os.makedirs(zone_path)
  77. svd_file_path = os.path.join(zone_path, output_svd_filename)
  78. # add writer into list
  79. svd_output_files.append(open(svd_file_path, 'w'))
  80. counter_index = 1
  81. while(counter_index < end_counter_index):
  82. if counter_index % picture_step == 0:
  83. counter_index_str = str(counter_index)
  84. if color:
  85. img_path = os.path.join(noise_path, folder_scene + "_" + noise + "_color_" + counter_index_str + ".png")
  86. else:
  87. img_path = os.path.join(noise_path, folder_scene + "_" + noise + "_" + counter_index_str + ".png")
  88. current_img = Image.open(img_path)
  89. img_blocks = processing.divide_in_blocks(current_img, (200, 200))
  90. for id_block, block in enumerate(img_blocks):
  91. ###########################
  92. # Metric computation part #
  93. ###########################
  94. data = get_svd_data(data_type, block)
  95. ##################
  96. # Data mode part #
  97. ##################
  98. # modify data depending mode
  99. if mode == 'svdne':
  100. # getting max and min information from min_max_filename
  101. with open(data_min_max_filename, 'r') as f:
  102. min_val = float(f.readline())
  103. max_val = float(f.readline())
  104. data = processing.normalize_arr_with_range(data, min_val, max_val)
  105. if mode == 'svdn':
  106. data = processing.normalize_arr(data)
  107. # save min and max found from dataset in order to normalize data using whole data known
  108. if mode == 'svd':
  109. current_min = data.min()
  110. current_max = data.max()
  111. if current_min < min_val_found:
  112. min_val_found = current_min
  113. if current_max > max_val_found:
  114. max_val_found = current_max
  115. # now write data into current writer
  116. current_file = svd_output_files[id_block]
  117. # add of index
  118. current_file.write(counter_index_str + ';')
  119. for val in data:
  120. current_file.write(str(val) + ";")
  121. current_file.write('\n')
  122. if color:
  123. print(data_type + "_" + noise + "_color_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((counter_index) / (end_counter_index)* 100.) + "%")
  124. else:
  125. print(data_type + "_" + noise + "_"+ mode + "_" + folder_scene + " - " + "{0:.2f}".format((counter_index) / (end_counter_index)* 100.) + "%")
  126. sys.stdout.write("\033[F")
  127. counter_index += 1
  128. for f in svd_output_files:
  129. f.close()
  130. if color:
  131. print(data_type + "_" + noise + "_color_" + mode + "_" + folder_scene + " - " + "Done...")
  132. else:
  133. print(data_type + "_" + noise + "_"+ mode + "_" + folder_scene + " - " + "Done...")
  134. # save current information about min file found
  135. if mode == 'svd':
  136. with open(data_min_max_filename, 'w') as f:
  137. f.write(str(min_val_found) + '\n')
  138. f.write(str(max_val_found) + '\n')
  139. print("%s : end of data generation\n" % mode)
  140. def main():
  141. # default value of p_step
  142. p_step = 10
  143. p_color = 0
  144. if len(sys.argv) <= 1:
  145. print('Run with default parameters...')
  146. print('python generate_all_data.py --metric all --color 0')
  147. print('python generate_all_data.py --metric lab --color 0')
  148. print('python generate_all_data.py --metric lab --color 1 --step 10')
  149. sys.exit(2)
  150. try:
  151. opts, args = getopt.getopt(sys.argv[1:], "hm:s:c", ["help=", "metric=", "step=", "color="])
  152. except getopt.GetoptError:
  153. # print help information and exit:
  154. print('python generate_all_data.py --metric all --color 1 --step 10')
  155. sys.exit(2)
  156. for o, a in opts:
  157. if o == "-h":
  158. print('python generate_all_data.py --metric all --color 1 --step 10')
  159. sys.exit()
  160. elif o in ("-s", "--step"):
  161. p_step = int(a)
  162. elif o in ("-c", "--color"):
  163. p_color = int(a)
  164. elif o in ("-m", "--metric"):
  165. p_metric = a
  166. if p_metric != 'all' and p_metric not in metric_choices:
  167. assert False, "Invalid metric choice"
  168. else:
  169. assert False, "unhandled option"
  170. global picture_step
  171. picture_step = p_step
  172. if picture_step % 10 != 0:
  173. assert False, "Picture step variable needs to be divided by ten"
  174. # generate all or specific metric data
  175. if p_metric == 'all':
  176. for m in metric_choices:
  177. generate_data_svd(m, p_color, 'svd')
  178. generate_data_svd(m, p_color, 'svdn')
  179. generate_data_svd(m, p_color, 'svdne')
  180. else:
  181. generate_data_svd(p_metric, p_color, 'svd')
  182. generate_data_svd(p_metric, p_color, 'svdn')
  183. generate_data_svd(p_metric, p_color, 'svdne')
  184. if __name__== "__main__":
  185. main()