generate_data_model_random.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Fri Sep 14 21:02:42 2018
  5. @author: jbuisine
  6. """
  7. from __future__ import print_function
  8. import sys, os, getopt
  9. import numpy as np
  10. import random
  11. import time
  12. import json
  13. from PIL import Image
  14. from ipfml import processing, metrics
  15. from modules.utils import config as cfg
  16. # getting configuration information
  17. config_filename = cfg.config_filename
  18. zone_folder = cfg.zone_folder
  19. min_max_filename = cfg.min_max_filename_extension
  20. # define all scenes values
  21. scenes_list = cfg.scenes_names
  22. scenes_indexes = cfg.scenes_indices
  23. choices = cfg.normalization_choices
  24. path = cfg.dataset_path
  25. zones = cfg.zones_indices
  26. seuil_expe_filename = cfg.seuil_expe_filename
  27. metric_choices = cfg.metric_choices_labels
  28. output_data_folder = cfg.output_data_folder
  29. custom_min_max_folder = cfg.min_max_custom_folder
  30. min_max_ext = cfg.min_max_filename_extension
  31. generic_output_file_svd = '_random.csv'
  32. min_value_interval = sys.maxsize
  33. max_value_interval = 0
  34. def construct_new_line(path_seuil, interval, line, norm, sep, index):
  35. begin, end = interval
  36. line_data = line.split(';')
  37. seuil = line_data[0]
  38. metrics = line_data[begin+1:end+1]
  39. metrics = [float(m) for m in metrics]
  40. # TODO : check if it's always necessary to do that (loss of information for svd)
  41. if norm:
  42. metrics = processing.normalize_arr_with_range(metrics, min_value_interval, max_value_interval)
  43. with open(path_seuil, "r") as seuil_file:
  44. seuil_learned = int(seuil_file.readline().strip())
  45. if seuil_learned > int(seuil):
  46. line = '1'
  47. else:
  48. line = '0'
  49. for idx, val in enumerate(metrics):
  50. if index:
  51. line += " " + str(idx + 1)
  52. line += sep
  53. line += str(val)
  54. line += '\n'
  55. return line
  56. def get_min_max_value_interval(_filename, _interval, _choice, _metric):
  57. global min_value_interval, max_value_interval
  58. scenes = os.listdir(path)
  59. # remove min max file from scenes folder
  60. scenes = [s for s in scenes if min_max_filename not in s]
  61. for id_scene, folder_scene in enumerate(scenes):
  62. # only take care of maxwell scenes
  63. if folder_scene in scenes_list:
  64. scene_path = os.path.join(path, folder_scene)
  65. zones_folder = []
  66. # create zones list
  67. for index in zones:
  68. index_str = str(index)
  69. if len(index_str) < 2:
  70. index_str = "0" + index_str
  71. zones_folder.append("zone"+index_str)
  72. # shuffle list of zones (=> randomly choose zones)
  73. random.shuffle(zones_folder)
  74. for id_zone, zone_folder in enumerate(zones_folder):
  75. zone_path = os.path.join(scene_path, zone_folder)
  76. data_filename = _metric + "_" + _choice + generic_output_file_svd
  77. data_file_path = os.path.join(zone_path, data_filename)
  78. # getting number of line and read randomly lines
  79. f = open(data_file_path)
  80. lines = f.readlines()
  81. counter = 0
  82. # check if user select current scene and zone to be part of training data set
  83. for line in lines:
  84. begin, end = _interval
  85. line_data = line.split(';')
  86. metrics = line_data[begin+1:end+1]
  87. metrics = [float(m) for m in metrics]
  88. min_value = min(metrics)
  89. max_value = max(metrics)
  90. if min_value < min_value_interval:
  91. min_value_interval = min_value
  92. if max_value > max_value_interval:
  93. max_value_interval = max_value
  94. counter += 1
  95. def generate_data_model(_filename, _interval, _choice, _metric, _scenes = scenes_list, _nb_zones = 4, _percent = 1, _norm = False, _sep=':', _index=True):
  96. output_train_filename = _filename + ".train"
  97. output_test_filename = _filename + ".test"
  98. if not '/' in output_train_filename:
  99. raise Exception("Please select filename with directory path to save data. Example : data/dataset")
  100. # create path if not exists
  101. if not os.path.exists(output_data_folder):
  102. os.makedirs(output_data_folder)
  103. train_file = open(output_train_filename, 'w')
  104. test_file = open(output_test_filename, 'w')
  105. scenes = os.listdir(path)
  106. # remove min max file from scenes folder
  107. scenes = [s for s in scenes if min_max_filename not in s]
  108. for id_scene, folder_scene in enumerate(scenes):
  109. # only take care of maxwell scenes
  110. if folder_scene in scenes_list:
  111. scene_path = os.path.join(path, folder_scene)
  112. zones_folder = []
  113. # create zones list
  114. for index in zones:
  115. index_str = str(index)
  116. if len(index_str) < 2:
  117. index_str = "0" + index_str
  118. zones_folder.append("zone"+index_str)
  119. # shuffle list of zones (=> randomly choose zones)
  120. random.shuffle(zones_folder)
  121. for id_zone, zone_folder in enumerate(zones_folder):
  122. zone_path = os.path.join(scene_path, zone_folder)
  123. data_filename = _metric + "_" + _choice + generic_output_file_svd
  124. data_file_path = os.path.join(zone_path, data_filename)
  125. # getting number of line and read randomly lines
  126. f = open(data_file_path)
  127. lines = f.readlines()
  128. num_lines = len(lines)
  129. lines_indexes = np.arange(num_lines)
  130. random.shuffle(lines_indexes)
  131. path_seuil = os.path.join(zone_path, seuil_expe_filename)
  132. counter = 0
  133. # check if user select current scene and zone to be part of training data set
  134. for index in lines_indexes:
  135. line = construct_new_line(path_seuil, _interval, lines[index], _norm, _sep, _index)
  136. percent = counter / num_lines
  137. if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
  138. train_file.write(line)
  139. else:
  140. test_file.write(line)
  141. counter += 1
  142. f.close()
  143. train_file.close()
  144. test_file.close()
  145. def main():
  146. p_custom = False
  147. if len(sys.argv) <= 1:
  148. print('Run with default parameters...')
  149. print('python generate_data_model_random.py --output xxxx --interval 0,20 --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1 --custom min_max_filename')
  150. sys.exit(2)
  151. try:
  152. opts, args = getopt.getopt(sys.argv[1:], "ho:i:k:s:n:p:r:c", ["help=", "output=", "interval=", "kind=", "metric=","scenes=", "nb_zones=", "percent=", "sep=", "rowindex=", "custom="])
  153. except getopt.GetoptError:
  154. # print help information and exit:
  155. print('python generate_data_model_random.py --output xxxx --interval 0,20 --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1 --custom min_max_filename')
  156. sys.exit(2)
  157. for o, a in opts:
  158. if o == "-h":
  159. print('python generate_data_model_random.py --output xxxx --interval 0,20 --kind svdne --metric lab --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1 --custom min_max_filename')
  160. sys.exit()
  161. elif o in ("-o", "--output"):
  162. p_filename = a
  163. elif o in ("-i", "--interval"):
  164. p_interval = list(map(int, a.split(',')))
  165. elif o in ("-k", "--kind"):
  166. p_kind = a
  167. elif o in ("-m", "--metric"):
  168. p_metric = a
  169. elif o in ("-s", "--scenes"):
  170. p_scenes = a.split(',')
  171. elif o in ("-n", "--nb_zones"):
  172. p_nb_zones = int(a)
  173. elif o in ("-p", "--percent"):
  174. p_percent = float(a)
  175. elif o in ("-s", "--sep"):
  176. p_sep = a
  177. elif o in ("-r", "--rowindex"):
  178. if int(a) == 1:
  179. p_rowindex = True
  180. else:
  181. p_rowindex = False
  182. elif o in ("-c", "--custom"):
  183. p_custom = a
  184. else:
  185. assert False, "unhandled option"
  186. # getting scenes from indexes user selection
  187. scenes_selected = []
  188. for scene_id in p_scenes:
  189. index = scenes_indexes.index(scene_id.strip())
  190. scenes_selected.append(scenes_list[index])
  191. # find min max value if necessary to renormalize data
  192. if p_custom:
  193. get_min_max_value_interval(p_filename, p_interval, p_kind, p_metric)
  194. # write new file to save
  195. if not os.path.exists(custom_min_max_folder):
  196. os.makedirs(custom_min_max_folder)
  197. min_max_folder_path = os.path.join(os.path.dirname(__file__), custom_min_max_folder)
  198. min_max_filename_path = os.path.join(min_max_folder_path, p_custom)
  199. with open(min_max_filename_path, 'w') as f:
  200. f.write(str(min_value_interval) + '\n')
  201. f.write(str(max_value_interval) + '\n')
  202. # create database using img folder (generate first time only)
  203. generate_data_model(p_filename, p_interval, p_kind, p_metric, scenes_selected, p_nb_zones, p_percent, p_custom, p_sep, p_rowindex)
  204. if __name__== "__main__":
  205. main()