generate_all_data_file.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225
  1. # main imports
  2. import sys, os, argparse
  3. import numpy as np
  4. import random
  5. import time
  6. import json
  7. # image processing imports
  8. from PIL import Image
  9. from ipfml.processing import transform, segmentation
  10. from ipfml import utils
  11. # modules imports
  12. sys.path.insert(0, '') # trick to enable import of main folder module
  13. import custom_config as cfg
  14. from modules.utils import data as dt
  15. from data_attributes import get_image_features
  16. # getting configuration information
  17. zone_folder = cfg.zone_folder
  18. min_max_filename = cfg.min_max_filename_extension
  19. # define all scenes values
  20. choices = cfg.normalization_choices
  21. zones = cfg.zones_indices
  22. seuil_expe_filename = cfg.seuil_expe_filename
  23. features_choices = cfg.features_choices_labels
  24. output_data_folder = cfg.output_data_folder
  25. generic_output_file_svd = '_random.csv'
  26. def generate_data_feature(path, output, human_thresholds, data_type, mode):
  27. """
  28. @brief Method which generates all .csv files from scenes
  29. @param data_type, feature choice
  30. @param mode, normalization choice
  31. @return nothing
  32. """
  33. scenes = os.listdir(path)
  34. # remove min max file from scenes folder
  35. scenes = [s for s in scenes if min_max_filename not in s]
  36. # keep in memory min and max data found from data_type
  37. min_val_found = sys.maxsize
  38. max_val_found = 0
  39. output_path = os.path.join(cfg.output_data_generated, output)
  40. if not os.path.exists(output_path):
  41. os.makedirs(output_path)
  42. data_min_max_filename = os.path.join(output_path, data_type + min_max_filename)
  43. # go ahead each scenes
  44. for folder_scene in human_thresholds:
  45. print(folder_scene)
  46. scene_path = os.path.join(path, folder_scene)
  47. output_scene_path = os.path.join(output_path, folder_scene)
  48. if not os.path.exists(output_scene_path):
  49. os.makedirs(output_scene_path)
  50. # getting output filename
  51. output_svd_filename = data_type + "_" + mode + generic_output_file_svd
  52. # construct each zones folder name
  53. zones_folder = []
  54. svd_output_files = []
  55. # get zones list info
  56. for index in zones:
  57. index_str = str(index)
  58. if len(index_str) < 2:
  59. index_str = "0" + index_str
  60. current_zone = "zone"+index_str
  61. zones_folder.append(current_zone)
  62. zone_path = os.path.join(scene_path, current_zone)
  63. output_zone_path = os.path.join(output_scene_path, current_zone)
  64. if not os.path.exists(output_zone_path):
  65. os.makedirs(output_zone_path)
  66. svd_file_path = os.path.join(output_zone_path, output_svd_filename)
  67. # add writer into list
  68. svd_output_files.append(open(svd_file_path, 'w'))
  69. # get all images of folder
  70. scene_images = sorted([os.path.join(scene_path, img) for img in os.listdir(scene_path) if cfg.scene_image_extension in img])
  71. number_scene_image = len(scene_images)
  72. for id_img, img_path in enumerate(scene_images):
  73. current_image_postfix = dt.get_scene_image_postfix(img_path)
  74. current_img = Image.open(img_path)
  75. img_blocks = segmentation.divide_in_blocks(current_img, (200, 200))
  76. for id_block, block in enumerate(img_blocks):
  77. ###########################
  78. # feature computation part #
  79. ###########################
  80. data = get_image_features(data_type, block)
  81. ##################
  82. # Data mode part #
  83. ##################
  84. # modify data depending mode
  85. if mode == 'svdne':
  86. # getting max and min information from min_max_filename
  87. with open(data_min_max_filename, 'r') as f:
  88. min_val = float(f.readline())
  89. max_val = float(f.readline())
  90. data = utils.normalize_arr_with_range(data, min_val, max_val)
  91. if mode == 'svdn':
  92. data = utils.normalize_arr(data)
  93. # save min and max found from dataset in order to normalize data using whole data known
  94. if mode == 'svd':
  95. current_min = data.min()
  96. current_max = data.max()
  97. if current_min < min_val_found:
  98. min_val_found = current_min
  99. if current_max > max_val_found:
  100. max_val_found = current_max
  101. # now write data into current writer
  102. current_file = svd_output_files[id_block]
  103. # add of index
  104. current_file.write(current_image_postfix + ';')
  105. for val in data:
  106. current_file.write(str(val) + ";")
  107. current_file.write('\n')
  108. print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((id_img + 1) / number_scene_image * 100.) + "%")
  109. sys.stdout.write("\033[F")
  110. for f in svd_output_files:
  111. f.close()
  112. print('\n')
  113. # save current information about min file found
  114. if mode == 'svd':
  115. with open(data_min_max_filename, 'w') as f:
  116. f.write(str(min_val_found) + '\n')
  117. f.write(str(max_val_found) + '\n')
  118. print("%s_%s : end of data generation\n" % (data_type, mode))
  119. def main():
  120. parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes (keep in memory min and max value found)")
  121. parser.add_argument('--feature', type=str,
  122. help="feature choice in order to compute data (use 'all' if all features are needed)", required=True)
  123. parser.add_argument('--dataset', type=str,
  124. help="dataset with all scenes", required=True)
  125. parser.add_argument('--output', type=str,
  126. help="output where data files are saved", required=True)
  127. parser.add_argument('--thresholds', type=str, help='file with scene list information and thresholds', required=True)
  128. args = parser.parse_args()
  129. p_feature = args.feature
  130. p_dataset = args.dataset
  131. p_output = args.output
  132. p_thresholds = args.thresholds
  133. # 1. retrieve human_thresholds
  134. human_thresholds = {}
  135. # extract thresholds
  136. with open(p_thresholds) as f:
  137. thresholds_line = f.readlines()
  138. for line in thresholds_line:
  139. data = line.split(';')
  140. del data[-1] # remove unused last element `\n`
  141. current_scene = data[0]
  142. thresholds_scene = data[1:]
  143. # TODO : check if really necessary
  144. if current_scene != '50_shades_of_grey':
  145. human_thresholds[current_scene] = [ int(threshold) for threshold in thresholds_scene ]
  146. # generate all or specific feature data
  147. if p_feature == 'all':
  148. for m in features_choices:
  149. generate_data_feature(p_dataset, p_output, human_thresholds, m, 'svd')
  150. generate_data_feature(p_dataset, p_output, human_thresholds, m, 'svdn')
  151. generate_data_feature(p_dataset, p_output, human_thresholds, m, 'svdne')
  152. else:
  153. if p_feature not in features_choices:
  154. raise ValueError('Unknown feature choice : ', features_choices)
  155. generate_data_feature(p_dataset, p_output, human_thresholds, p_feature, 'svd')
  156. generate_data_feature(p_dataset, p_output, human_thresholds, p_feature, 'svdn')
  157. generate_data_feature(p_dataset, p_output, human_thresholds, p_feature, 'svdne')
  158. if __name__== "__main__":
  159. main()