generate_all_data.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Fri Sep 14 21:02:42 2018
  5. @author: jbuisine
  6. """
  7. from __future__ import print_function
  8. import sys, os, argparse
  9. import numpy as np
  10. import random
  11. import time
  12. import json
  13. from modules.utils.data import get_svd_data
  14. from PIL import Image
  15. from ipfml import processing, metrics, utils
  16. from skimage import color
  17. from modules.utils import config as cfg
  18. # getting configuration information
  19. config_filename = cfg.config_filename
  20. zone_folder = cfg.zone_folder
  21. min_max_filename = cfg.min_max_filename_extension
  22. # define all scenes values
  23. scenes_list = cfg.scenes_names
  24. scenes_indexes = cfg.scenes_indices
  25. choices = cfg.normalization_choices
  26. path = cfg.dataset_path
  27. zones = cfg.zones_indices
  28. seuil_expe_filename = cfg.seuil_expe_filename
  29. metric_choices = cfg.metric_choices_labels
  30. output_data_folder = cfg.output_data_folder
  31. generic_output_file_svd = '_random.csv'
  32. def generate_data_svd(data_type, mode):
  33. """
  34. @brief Method which generates all .csv files from scenes
  35. @param data_type, metric choice
  36. @param mode, normalization choice
  37. @return nothing
  38. """
  39. scenes = os.listdir(path)
  40. # remove min max file from scenes folder
  41. scenes = [s for s in scenes if min_max_filename not in s]
  42. # keep in memory min and max data found from data_type
  43. min_val_found = sys.maxsize
  44. max_val_found = 0
  45. data_min_max_filename = os.path.join(path, data_type + min_max_filename)
  46. # go ahead each scenes
  47. for id_scene, folder_scene in enumerate(scenes):
  48. print(folder_scene)
  49. scene_path = os.path.join(path, folder_scene)
  50. config_file_path = os.path.join(scene_path, config_filename)
  51. with open(config_file_path, "r") as config_file:
  52. last_image_name = config_file.readline().strip()
  53. prefix_image_name = config_file.readline().strip()
  54. start_index_image = config_file.readline().strip()
  55. end_index_image = config_file.readline().strip()
  56. step_counter = int(config_file.readline().strip())
  57. # getting output filename
  58. output_svd_filename = data_type + "_" + mode + generic_output_file_svd
  59. # construct each zones folder name
  60. zones_folder = []
  61. svd_output_files = []
  62. # get zones list info
  63. for index in zones:
  64. index_str = str(index)
  65. if len(index_str) < 2:
  66. index_str = "0" + index_str
  67. current_zone = "zone"+index_str
  68. zones_folder.append(current_zone)
  69. zone_path = os.path.join(scene_path, current_zone)
  70. svd_file_path = os.path.join(zone_path, output_svd_filename)
  71. # add writer into list
  72. svd_output_files.append(open(svd_file_path, 'w'))
  73. current_counter_index = int(start_index_image)
  74. end_counter_index = int(end_index_image)
  75. while(current_counter_index <= end_counter_index):
  76. current_counter_index_str = str(current_counter_index)
  77. while len(start_index_image) > len(current_counter_index_str):
  78. current_counter_index_str = "0" + current_counter_index_str
  79. img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
  80. current_img = Image.open(img_path)
  81. img_blocks = processing.divide_in_blocks(current_img, (200, 200))
  82. for id_block, block in enumerate(img_blocks):
  83. ###########################
  84. # Metric computation part #
  85. ###########################
  86. data = get_svd_data(data_type, block)
  87. ##################
  88. # Data mode part #
  89. ##################
  90. # modify data depending mode
  91. if mode == 'svdne':
  92. # getting max and min information from min_max_filename
  93. with open(data_min_max_filename, 'r') as f:
  94. min_val = float(f.readline())
  95. max_val = float(f.readline())
  96. data = utils.normalize_arr_with_range(data, min_val, max_val)
  97. if mode == 'svdn':
  98. data = utils.normalize_arr(data)
  99. # save min and max found from dataset in order to normalize data using whole data known
  100. if mode == 'svd':
  101. current_min = data.min()
  102. current_max = data.max()
  103. if current_min < min_val_found:
  104. min_val_found = current_min
  105. if current_max > max_val_found:
  106. max_val_found = current_max
  107. # now write data into current writer
  108. current_file = svd_output_files[id_block]
  109. # add of index
  110. current_file.write(current_counter_index_str + ';')
  111. for val in data:
  112. current_file.write(str(val) + ";")
  113. current_file.write('\n')
  114. start_index_image_int = int(start_index_image)
  115. print(data_type + "_" + mode + "_" + folder_scene + " - " + "{0:.2f}".format((current_counter_index - start_index_image_int) / (end_counter_index - start_index_image_int)* 100.) + "%")
  116. sys.stdout.write("\033[F")
  117. current_counter_index += step_counter
  118. for f in svd_output_files:
  119. f.close()
  120. print('\n')
  121. # save current information about min file found
  122. if mode == 'svd':
  123. with open(data_min_max_filename, 'w') as f:
  124. f.write(str(min_val_found) + '\n')
  125. f.write(str(max_val_found) + '\n')
  126. print("%s_%s : end of data generation\n" % (data_type, mode))
  127. def main():
  128. parser = argparse.ArgumentParser(description="Compute and prepare data of metric of all scenes (keep in memory min and max value found)")
  129. parser.add_argument('--metric', type=str,
  130. help="metric choice in order to compute data (use 'all' if all metrics are needed)",
  131. choices=metric_choices)
  132. args = parser.parse_args()
  133. p_metric = args.metric
  134. # generate all or specific metric data
  135. if p_metric == 'all':
  136. for m in metric_choices:
  137. generate_data_svd(m, 'svd')
  138. generate_data_svd(m, 'svdn')
  139. generate_data_svd(m, 'svdne')
  140. else:
  141. generate_data_svd(p_metric, 'svd')
  142. generate_data_svd(p_metric, 'svdn')
  143. generate_data_svd(p_metric, 'svdne')
  144. if __name__== "__main__":
  145. main()