generate_data_model_random.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Fri Sep 14 21:02:42 2018
  5. @author: jbuisine
  6. """
  7. from __future__ import print_function
  8. import sys, os, getopt
  9. import numpy as np
  10. import random
  11. import time
  12. import json
  13. from PIL import Image
  14. from ipfml import processing, metrics, utils
  15. from modules.utils import config as cfg
  16. from modules.utils import data as dt
  17. # getting configuration information
  18. config_filename = cfg.config_filename
  19. zone_folder = cfg.zone_folder
  20. min_max_filename = cfg.min_max_filename_extension
  21. # define all scenes values
  22. all_scenes_list = cfg.scenes_names
  23. all_scenes_indices = cfg.scenes_indices
  24. normalization_choices = cfg.normalization_choices
  25. path = cfg.dataset_path
  26. zones = cfg.zones_indices
  27. seuil_expe_filename = cfg.seuil_expe_filename
  28. metric_choices = cfg.metric_choices_labels
  29. output_data_folder = cfg.output_data_folder
  30. custom_min_max_folder = cfg.min_max_custom_folder
  31. min_max_ext = cfg.min_max_filename_extension
  32. generic_output_file_svd = '_random.csv'
  33. min_value_interval = sys.maxsize
  34. max_value_interval = 0
  35. def construct_new_line(path_seuil, interval, line, choice, norm):
  36. begin, end = interval
  37. line_data = line.split(';')
  38. seuil = line_data[0]
  39. metrics = line_data[begin+1:end+1]
  40. metrics = [float(m) for m in metrics]
  41. # TODO : check if it's always necessary to do that (loss of information for svd)
  42. if norm:
  43. if choice == 'svdne':
  44. metrics = utils.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
  45. if choice == 'svdn':
  46. metrics = utils.normalize_arr(metrics)
  47. with open(path_seuil, "r") as seuil_file:
  48. seuil_learned = int(seuil_file.readline().strip())
  49. if seuil_learned > int(seuil):
  50. line = '1'
  51. else:
  52. line = '0'
  53. for idx, val in enumerate(metrics):
  54. line += ';'
  55. line += str(val)
  56. line += '\n'
  57. return line
  58. def get_min_max_value_interval(_scenes_list, _filename, _interval, _choice, _metric, _custom):
  59. global min_value_interval, max_value_interval
  60. scenes = os.listdir(path)
  61. # remove min max file from scenes folder
  62. scenes = [s for s in scenes if min_max_filename not in s]
  63. for id_scene, folder_scene in enumerate(scenes):
  64. # only take care of maxwell scenes
  65. if folder_scene in _scenes_list:
  66. scene_path = os.path.join(path, folder_scene)
  67. zones_folder = []
  68. # create zones list
  69. for index in zones:
  70. index_str = str(index)
  71. if len(index_str) < 2:
  72. index_str = "0" + index_str
  73. zones_folder.append("zone"+index_str)
  74. # shuffle list of zones (=> randomly choose zones)
  75. random.shuffle(zones_folder)
  76. for id_zone, zone_folder in enumerate(zones_folder):
  77. zone_path = os.path.join(scene_path, zone_folder)
  78. # if custom normalization choices then we use svd values not already normalized
  79. if _custom:
  80. data_filename = _metric + "_svd"+ generic_output_file_svd
  81. else:
  82. data_filename = _metric + "_" + _choice + generic_output_file_svd
  83. data_file_path = os.path.join(zone_path, data_filename)
  84. # getting number of line and read randomly lines
  85. f = open(data_file_path)
  86. lines = f.readlines()
  87. counter = 0
  88. # check if user select current scene and zone to be part of training data set
  89. for line in lines:
  90. begin, end = _interval
  91. line_data = line.split(';')
  92. metrics = line_data[begin+1:end+1]
  93. metrics = [float(m) for m in metrics]
  94. min_value = min(metrics)
  95. max_value = max(metrics)
  96. if min_value < min_value_interval:
  97. min_value_interval = min_value
  98. if max_value > max_value_interval:
  99. max_value_interval = max_value
  100. counter += 1
  101. def generate_data_model(_scenes_list, _filename, _interval, _choice, _metric, _scenes, _nb_zones = 4, _percent = 1, _random=0, _step=1, _custom = False):
  102. output_train_filename = _filename + ".train"
  103. output_test_filename = _filename + ".test"
  104. if not '/' in output_train_filename:
  105. raise Exception("Please select filename with directory path to save data. Example : data/dataset")
  106. # create path if not exists
  107. if not os.path.exists(output_data_folder):
  108. os.makedirs(output_data_folder)
  109. scenes = os.listdir(path)
  110. # remove min max file from scenes folder
  111. scenes = [s for s in scenes if min_max_filename not in s]
  112. train_file_data = []
  113. test_file_data = []
  114. for id_scene, folder_scene in enumerate(scenes):
  115. # only take care of maxwell scenes
  116. if folder_scene in _scenes_list:
  117. scene_path = os.path.join(path, folder_scene)
  118. zones_folder = []
  119. # create zones list
  120. for index in zones:
  121. index_str = str(index)
  122. if len(index_str) < 2:
  123. index_str = "0" + index_str
  124. zones_folder.append("zone"+index_str)
  125. # shuffle list of zones (=> randomly choose zones)
  126. # only in random mode
  127. if _random:
  128. random.shuffle(zones_folder)
  129. for id_zone, zone_folder in enumerate(zones_folder):
  130. zone_path = os.path.join(scene_path, zone_folder)
  131. # if custom normalization choices then we use svd values not already normalized
  132. if _custom:
  133. data_filename = _metric + "_svd"+ generic_output_file_svd
  134. else:
  135. data_filename = _metric + "_" + _choice + generic_output_file_svd
  136. data_file_path = os.path.join(zone_path, data_filename)
  137. # getting number of line and read randomly lines
  138. f = open(data_file_path)
  139. lines = f.readlines()
  140. num_lines = len(lines)
  141. # randomly shuffle image
  142. if _random:
  143. random.shuffle(lines)
  144. path_seuil = os.path.join(zone_path, seuil_expe_filename)
  145. counter = 0
  146. # check if user select current scene and zone to be part of training data set
  147. for data in lines:
  148. percent = counter / num_lines
  149. image_index = int(data.split(';')[0])
  150. if image_index % _step == 0:
  151. line = construct_new_line(path_seuil, _interval, data, _choice, _custom)
  152. if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
  153. train_file_data.append(line)
  154. else:
  155. test_file_data.append(line)
  156. counter += 1
  157. f.close()
  158. train_file = open(output_train_filename, 'w')
  159. test_file = open(output_test_filename, 'w')
  160. for line in train_file_data:
  161. train_file.write(line)
  162. for line in test_file_data:
  163. test_file.write(line)
  164. train_file.close()
  165. test_file.close()
  166. def main():
  167. p_custom = False
  168. p_step = 1
  169. p_renderer = 'all'
  170. if len(sys.argv) <= 1:
  171. print('Run with default parameters...')
  172. print('python generate_data_model.py --output xxxx --interval 0,20 --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --random 1 --percent 0.7 --step 10 renderer all --custom min_max_filename')
  173. sys.exit(2)
  174. try:
  175. opts, args = getopt.getopt(sys.argv[1:], "ho:i:k:s:n:r:p:s:r:c", ["help=", "output=", "interval=", "kind=", "metric=","scenes=", "nb_zones=", "random=", "percent=", "step=", "renderer=", "custom="])
  176. except getopt.GetoptError:
  177. # print help information and exit:
  178. print('python generate_data_model.py --output xxxx --interval 0,20 --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --random 1 --percent 0.7 --step 10 --renderer all --custom min_max_filename')
  179. sys.exit(2)
  180. for o, a in opts:
  181. if o == "-h":
  182. print('python generate_data_model.py --output xxxx --interval 0,20 --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --random 1 --percent 0.7 --step 10 --renderer all --custom min_max_filename')
  183. sys.exit()
  184. elif o in ("-o", "--output"):
  185. p_filename = a
  186. elif o in ("-i", "--interval"):
  187. p_interval = list(map(int, a.split(',')))
  188. elif o in ("-k", "--kind"):
  189. p_kind = a
  190. if p_kind not in normalization_choices:
  191. assert False, "Invalid normalization choice, %s" % normalization_choices
  192. elif o in ("-m", "--metric"):
  193. p_metric = a
  194. elif o in ("-s", "--scenes"):
  195. p_scenes = a.split(',')
  196. elif o in ("-n", "--nb_zones"):
  197. p_nb_zones = int(a)
  198. elif o in ("-r", "--random"):
  199. p_random = int(a)
  200. elif o in ("-p", "--percent"):
  201. p_percent = float(a)
  202. elif o in ("-s", "--sep"):
  203. p_sep = a
  204. elif o in ("-s", "--step"):
  205. p_step = int(a)
  206. elif o in ("-r", "--renderer"):
  207. p_renderer = a
  208. if p_renderer not in cfg.renderer_choices:
  209. assert False, "Unknown renderer choice, %s" % cfg.renderer_choices
  210. elif o in ("-c", "--custom"):
  211. p_custom = a
  212. else:
  213. assert False, "unhandled option"
  214. # list all possibles choices of renderer
  215. scenes_list = dt.get_renderer_scenes_names(p_renderer)
  216. scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
  217. # getting scenes from indexes user selection
  218. scenes_selected = []
  219. for scene_id in p_scenes:
  220. index = scenes_indices.index(scene_id.strip())
  221. scenes_selected.append(scenes_list[index])
  222. # find min max value if necessary to renormalize data
  223. if p_custom:
  224. get_min_max_value_interval(scenes_list, p_filename, p_interval, p_kind, p_metric, p_custom)
  225. # write new file to save
  226. if not os.path.exists(custom_min_max_folder):
  227. os.makedirs(custom_min_max_folder)
  228. min_max_folder_path = os.path.join(os.path.dirname(__file__), custom_min_max_folder)
  229. min_max_filename_path = os.path.join(min_max_folder_path, p_custom)
  230. with open(min_max_filename_path, 'w') as f:
  231. f.write(str(min_value_interval) + '\n')
  232. f.write(str(max_value_interval) + '\n')
  233. # create database using img folder (generate first time only)
  234. generate_data_model(scenes_list, p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_random, p_step, p_custom)
  235. if __name__== "__main__":
  236. main()