generate_data_svm_random.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Fri Sep 14 21:02:42 2018
  5. @author: jbuisine
  6. """
  7. from __future__ import print_function
  8. import sys, os, getopt
  9. import numpy as np
  10. import random
  11. import time
  12. import json
  13. config_filename = "config"
  14. zone_folder = "zone"
  15. min_max_filename = "min_max_values"
  16. output_file_svd = "SVD_LAB_test_im6.csv"
  17. output_file_svdn = "SVDN_LAB_test_im6.csv"
  18. output_file_svdne = "SVDNE_LAB_test_im6.csv"
  19. # define all scenes values
  20. scenes = ['Appart1opt02', 'Bureau1', 'Cendrier', 'Cuisine01', 'EchecsBas', 'PNDVuePlongeante', 'SdbCentre', 'SdbDroite', 'Selles']
  21. scenes_indexes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
  22. choices = ['svd', 'svdn', 'svdne']
  23. path = './fichiersSVD'
  24. zones = np.arange(16)
  25. file_choice = [output_file_svd, output_file_svdn, output_file_svdne]
  26. seuil_expe_filename = 'seuilExpe'
  27. def generate_data_svd_lab():
  28. """
  29. @brief Method which generates all .csv files from scenes photos
  30. @param path - path of scenes folder information
  31. @return nothing
  32. """
  33. # TODO :
  34. # - parcourir chaque dossier de scene
  35. scenes = os.listdir(path)
  36. for folder_scene in scenes:
  37. folder_path = path + "/" + folder_scene
  38. with open(folder_path + "/" + config_filename, "r") as config_file:
  39. last_image_name = config_file.readline().strip()
  40. prefix_image_name = config_file.readline().strip()
  41. start_index_image = config_file.readline().strip()
  42. end_index_image = config_file.readline().strip()
  43. step_counter = int(config_file.readline().strip())
  44. current_counter_index = int(start_index_image)
  45. end_counter_index = int(start_index_image)
  46. print(current_counter_index)
  47. while(current_counter_index <= end_index_image):
  48. print(current_counter_index)
  49. current_counter_index += step_counter
  50. # - récupérer les informations des fichiers de configurations
  51. # - création des fichiers de sortie SVD, SVDE, SVDNE
  52. def construct_new_line(path_seuil, interval, line, sep, index):
  53. begin, end = interval
  54. line_data = line.split(';')
  55. seuil = line_data[0]
  56. metrics = line_data[begin+1:end+1]
  57. with open(path_seuil, "r") as seuil_file:
  58. seuil_learned = int(seuil_file.readline().strip())
  59. if seuil_learned > int(seuil):
  60. line = '0'
  61. else:
  62. line = '1'
  63. for idx, val in enumerate(metrics):
  64. if index:
  65. line += " " + str(idx + 1)
  66. line += sep
  67. line += val
  68. line += '\n'
  69. return line
  70. def generate_data_svm(_filename, _interval, _choice, _scenes = scenes, _nb_zones = 4, _percent = 1, _sep=':', _index=True):
  71. output_train_filename = _filename + ".train"
  72. output_test_filename = _filename + ".test"
  73. if not '/' in output_train_filename:
  74. raise Exception("Please select filename with directory path to save data. Example : data/dataset")
  75. # create path if not exists
  76. output_folder = output_train_filename.split('/')[0]
  77. if not os.path.exists(output_folder):
  78. os.makedirs(output_folder)
  79. train_file = open(output_train_filename, 'w')
  80. test_file = open(output_test_filename, 'w')
  81. scenes = os.listdir(path)
  82. for id_scene, folder_scene in enumerate(scenes):
  83. scene_path = path + "/" + folder_scene
  84. zones_folder = []
  85. # create zones list
  86. for index in zones:
  87. index_str = str(index)
  88. if len(index_str) < 2:
  89. index_str = "0" + index_str
  90. zones_folder.append("zone"+index_str)
  91. # shuffle list of zones (=> randomly choose zones)
  92. random.shuffle(zones_folder)
  93. for id_zone, zone_folder in enumerate(zones_folder):
  94. zone_path = scene_path + "/" + zone_folder
  95. data_filename = file_choice[choices.index(_choice)]
  96. data_file_path = zone_path + "/" + data_filename
  97. # getting number of line and read randomly lines
  98. f = open(data_file_path)
  99. lines = f.readlines()
  100. num_lines = len(lines)
  101. lines_indexes = np.arange(num_lines)
  102. random.shuffle(lines_indexes)
  103. path_seuil = zone_path + "/" + seuil_expe_filename
  104. counter = 0
  105. # check if user select current scene and zone to be part of training data set
  106. for index in lines_indexes:
  107. line = construct_new_line(path_seuil, _interval, lines[index], _sep, _index)
  108. percent = counter / num_lines
  109. if id_zone < _nb_zones and folder_scene in _scenes and percent <= _percent:
  110. train_file.write(line)
  111. else:
  112. test_file.write(line)
  113. counter += 1
  114. f.close()
  115. train_file.close()
  116. test_file.close()
  117. def main():
  118. if len(sys.argv) <= 1:
  119. print('Run with default parameters...')
  120. print('python generate_data.py --output xxxx --interval 0,20 --kind svdne --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1')
  121. sys.exit(2)
  122. try:
  123. opts, args = getopt.getopt(sys.argv[1:], "ho:i:k:s:n:p:r", ["help=", "output=", "interval=", "kind=", "scenes=", "nb_zones=", "percent=", "sep=", "rowindex="])
  124. except getopt.GetoptError:
  125. # print help information and exit:
  126. print('python generate_data.py --output xxxx --interval 0,20 --kind svdne --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1')
  127. sys.exit(2)
  128. for o, a in opts:
  129. if o == "-h":
  130. print('python generate_data.py --output xxxx --interval 0,20 --kind svdne --scenes "A, B, D" --nb_zones 5 --percent 0.7 --sep : --rowindex 1')
  131. sys.exit()
  132. elif o in ("-o", "--output"):
  133. p_filename = a
  134. elif o in ("-i", "--interval"):
  135. p_interval = list(map(int, a.split(',')))
  136. elif o in ("-k", "--kind"):
  137. p_kind = a
  138. elif o in ("-s", "--scenes"):
  139. p_scenes = a.split(',')
  140. elif o in ("-n", "--nb_zones"):
  141. p_nb_zones = int(a)
  142. elif o in ("-p", "--percent"):
  143. p_percent = float(a)
  144. elif o in ("-s", "--sep"):
  145. p_sep = a
  146. elif o in ("-r", "--rowindex"):
  147. if int(a) == 1:
  148. p_rowindex = True
  149. else:
  150. p_rowindex = False
  151. else:
  152. assert False, "unhandled option"
  153. # getting scenes from indexes user selection
  154. scenes_selected = []
  155. for scene_id in p_scenes:
  156. index = scenes_indexes.index(scene_id.strip())
  157. scenes_selected.append(scenes[index])
  158. for scene in scenes_selected:
  159. print(scene)
  160. # create database using img folder (generate first time only)
  161. generate_data_svm(p_filename, p_interval, p_kind, scenes_selected, p_nb_zones, p_percent, p_sep, p_rowindex)
  162. if __name__== "__main__":
  163. main()