generate_all_data.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Fri Sep 14 21:02:42 2018
  5. @author: jbuisine
  6. """
  7. from __future__ import print_function
  8. import sys, os, getopt
  9. import numpy as np
  10. import random
  11. import time
  12. import json
  13. from PIL import Image
  14. from ipfml import image_processing
  15. from ipfml import metrics
  16. from skimage import color
  17. config_filename = "config"
  18. zone_folder = "zone"
  19. min_max_filename = "_min_max_values"
  20. generic_output_file_svd = '_random.csv'
  21. output_data_folder = 'data'
  22. # define all scenes values
  23. scenes = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
  24. scenes_indexes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
  25. choices = ['svd', 'svdn', 'svdne']
  26. path = './fichiersSVD_light'
  27. zones = np.arange(16)
  28. seuil_expe_filename = 'seuilExpe'
  29. metric_choices = ['mscn', 'mscn_revisited', 'low_bits_2', 'low_bits_3', 'low_bits_4']
  30. def generate_data_svd(data_type, mode):
  31. """
  32. @brief Method which generates all .csv files from scenes photos
  33. @param path - path of scenes folder information
  34. @return nothing
  35. """
  36. scenes = os.listdir(path)
  37. # remove min max file from scenes folder
  38. scenes = [s for s in scenes if min_max_filename not in s]
  39. # keep in memory min and max data found from data_type
  40. min_val_found = 100000000000
  41. max_val_found = 0
  42. data_min_max_filename = os.path.join(path, data_type + min_max_filename)
  43. # go ahead each scenes
  44. for id_scene, folder_scene in enumerate(scenes):
  45. print(folder_scene)
  46. scene_path = os.path.join(path, folder_scene)
  47. config_file_path = os.path.join(scene_path, config_filename)
  48. with open(config_file_path, "r") as config_file:
  49. last_image_name = config_file.readline().strip()
  50. prefix_image_name = config_file.readline().strip()
  51. start_index_image = config_file.readline().strip()
  52. end_index_image = config_file.readline().strip()
  53. step_counter = int(config_file.readline().strip())
  54. # getting output filename
  55. output_svd_filename = data_type + "_" + mode + generic_output_file_svd
  56. # construct each zones folder name
  57. zones_folder = []
  58. svd_output_files = []
  59. # get zones list info
  60. for index in zones:
  61. index_str = str(index)
  62. if len(index_str) < 2:
  63. index_str = "0" + index_str
  64. current_zone = "zone"+index_str
  65. zones_folder.append(current_zone)
  66. zone_path = os.path.join(scene_path, current_zone)
  67. svd_file_path = os.path.join(zone_path, output_svd_filename)
  68. # add writer into list
  69. svd_output_files.append(open(svd_file_path, 'w'))
  70. current_counter_index = int(start_index_image)
  71. end_counter_index = int(end_index_image)
  72. while(current_counter_index <= end_counter_index):
  73. current_counter_index_str = str(current_counter_index)
  74. while len(start_index_image) > len(current_counter_index_str):
  75. current_counter_index_str = "0" + current_counter_index_str
  76. img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
  77. current_img = Image.open(img_path)
  78. img_blocks = image_processing.divide_in_blocks(current_img, (200, 200))
  79. for id_block, block in enumerate(img_blocks):
  80. ###########################
  81. # Metric computation part #
  82. ###########################
  83. # get data from mode
  84. # Here you can add the way you compute data
  85. if data_type == 'lab':
  86. block_file_path = '/tmp/lab_img.png'
  87. block.save(block_file_path)
  88. data = image_processing.get_LAB_L_SVD_s(Image.open(block_file_path))
  89. if data_type == 'mscn_revisited':
  90. img_mscn_revisited = image_processing.rgb_to_mscn(block)
  91. # save tmp as img
  92. img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L')
  93. mscn_revisited_file_path = '/tmp/mscn_revisited_img.png'
  94. img_output.save(mscn_revisited_file_path)
  95. img_block = Image.open(mscn_revisited_file_path)
  96. # extract from temp image
  97. data = metrics.get_SVD_s(img_block)
  98. if data_type == 'mscn':
  99. img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8')
  100. img_mscn = image_processing.calculate_mscn_coefficients(img_gray, 7)
  101. img_mscn_norm = image_processing.normalize_2D_arr(img_mscn)
  102. img_mscn_gray = np.array(img_mscn_norm*255, 'uint8')
  103. data = metrics.get_SVD_s(img_mscn_gray)
  104. if data_type == 'low_bits_4':
  105. low_bits_4 = image_processing.rgb_to_LAB_L_low_bits(block)
  106. # extract from temp image
  107. data = metrics.get_SVD_s(low_bits_4)
  108. if data_type == 'low_bits_3':
  109. low_bits_3 = image_processing.rgb_to_LAB_L_low_bits(block, 7)
  110. # extract from temp image
  111. data = metrics.get_SVD_s(low_bits_3)
  112. if data_type == 'low_bits_2':
  113. low_bits_2 = image_processing.rgb_to_LAB_L_low_bits(block, 3)
  114. # extract from temp image
  115. data = metrics.get_SVD_s(low_bits_2)
  116. ##################
  117. # Data mode part #
  118. ##################
  119. # modify data depending mode
  120. if mode == 'svdne':
  121. # getting max and min information from min_max_filename
  122. with open(data_min_max_filename, 'r') as f:
  123. min_val = float(f.readline())
  124. max_val = float(f.readline())
  125. data = image_processing.normalize_arr_with_range(data, min_val, max_val)
  126. if mode == 'svdn':
  127. data = image_processing.normalize_arr(data)
  128. # save min and max found from dataset in order to normalize data using whole data known
  129. if mode == 'svd':
  130. current_min = data.min()
  131. current_max = data.max()
  132. if current_min < min_val_found:
  133. min_val_found = current_min
  134. if current_max > max_val_found:
  135. max_val_found = current_max
  136. # now write data into current writer
  137. current_file = svd_output_files[id_block]
  138. # add of index
  139. current_file.write(current_counter_index_str + ';')
  140. for val in data:
  141. current_file.write(str(val) + ";")
  142. current_file.write('\n')
  143. start_index_image_int = int(start_index_image)
  144. print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((current_counter_index - start_index_image_int) / (end_counter_index - start_index_image_int)* 100.) + "%")
  145. current_counter_index += step_counter
  146. for f in svd_output_files:
  147. f.close()
  148. # save current information about min file found
  149. if mode == 'svd':
  150. with open(data_min_max_filename, 'w') as f:
  151. f.write(str(min_val_found) + '\n')
  152. f.write(str(max_val_found) + '\n')
  153. print("End of data generation")
  154. def main():
  155. if len(sys.argv) <= 1:
  156. print('Run with default parameters...')
  157. print('python generate_all_data.py --metric all')
  158. print('python generate_all_data.py --metric lab')
  159. sys.exit(2)
  160. try:
  161. opts, args = getopt.getopt(sys.argv[1:], "hm", ["help=", "metric="])
  162. except getopt.GetoptError:
  163. # print help information and exit:
  164. print('python generate_all_data.py --metric all')
  165. sys.exit(2)
  166. for o, a in opts:
  167. if o == "-h":
  168. print('python generate_all_data.py --metric all')
  169. sys.exit()
  170. elif o in ("-m", "--metric"):
  171. p_metric = a
  172. if p_metric != 'all' and p_metric not in metric_choices:
  173. assert False, "Invalid metric choice"
  174. else:
  175. assert False, "unhandled option"
  176. # generate all or specific metric data
  177. if p_metric == 'all':
  178. for m in metric_choices:
  179. generate_data_svd(m, 'svd')
  180. generate_data_svd(m, 'svdn')
  181. generate_data_svd(m, 'svdne')
  182. else:
  183. generate_data_svd(p_metric, 'svd')
  184. generate_data_svd(p_metric, 'svdn')
  185. generate_data_svd(p_metric, 'svdne')
  186. if __name__== "__main__":
  187. main()