generate_all_data_augmentation.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166
  1. # main imports
  2. import sys, os, argparse
  3. import numpy as np
  4. import random
  5. import time
  6. import json
  7. # image processing imports
  8. from PIL import Image
  9. from ipfml.processing import transform, segmentation
  10. from ipfml import utils
  11. # modules imports
  12. sys.path.insert(0, '') # trick to enable import of main folder module
  13. import custom_config as cfg
  14. from modules.utils import data as dt
  15. from data_attributes import get_image_features
  16. # getting configuration information
  17. zone_folder = cfg.zone_folder
  18. min_max_filename = cfg.min_max_filename_extension
  19. # define all scenes values
  20. scenes_list = cfg.scenes_names
  21. scenes_indexes = cfg.scenes_indices
  22. choices = cfg.normalization_choices
  23. zones = cfg.zones_indices
  24. seuil_expe_filename = cfg.seuil_expe_filename
  25. features_choices = cfg.features_choices_labels
  26. output_data_folder = cfg.output_data_folder
  27. data_augmented_filename = cfg.data_augmented_filename
  28. generic_output_file_svd = '_random.csv'
  29. def generate_data_svd(data_type, mode, path):
  30. """
  31. @brief Method which generates all .csv files from scenes
  32. @param data_type, feature choice
  33. @param mode, normalization choice
  34. @param path, data augmented path
  35. @return nothing
  36. """
  37. scenes = os.listdir(path)
  38. # remove min max file from scenes folder
  39. scenes = [s for s in scenes if min_max_filename and generic_output_file_svd not in s]
  40. # keep in memory min and max data found from data_type
  41. min_val_found = sys.maxsize
  42. max_val_found = 0
  43. data_min_max_filename = os.path.join(path, data_type + min_max_filename)
  44. data_filename = os.path.join(path, data_augmented_filename)
  45. # getting output filename
  46. output_svd_filename = data_type + "_" + mode + generic_output_file_svd
  47. current_file = open(os.path.join(path, output_svd_filename), 'w')
  48. with open(data_filename, 'r') as f:
  49. lines = f.readlines()
  50. number_of_images = len(lines)
  51. for index, line in enumerate(lines):
  52. data = line.split(';')
  53. scene_name = data[0]
  54. number_of_samples = data[2]
  55. label_img = data[3]
  56. img_path = data[4].replace('\n', '')
  57. block = Image.open(os.path.join(path, img_path))
  58. ###########################
  59. # feature computation part #
  60. ###########################
  61. data = get_image_features(data_type, block)
  62. ##################
  63. # Data mode part #
  64. ##################
  65. # modify data depending mode
  66. if mode == 'svdne':
  67. # getting max and min information from min_max_filename
  68. with open(data_min_max_filename, 'r') as f:
  69. min_val = float(f.readline())
  70. max_val = float(f.readline())
  71. data = utils.normalize_arr_with_range(data, min_val, max_val)
  72. if mode == 'svdn':
  73. data = utils.normalize_arr(data)
  74. # save min and max found from dataset in order to normalize data using whole data known
  75. if mode == 'svd':
  76. current_min = data.min()
  77. current_max = data.max()
  78. if current_min < min_val_found:
  79. min_val_found = current_min
  80. if current_max > max_val_found:
  81. max_val_found = current_max
  82. # add of index
  83. current_file.write(scene_name + ';' + number_of_samples + ';' + label_img + ';')
  84. for val in data:
  85. current_file.write(str(val) + ";")
  86. print(data_type + "_" + mode + " - " + "{0:.2f}".format((index + 1) / number_of_images * 100.) + "%")
  87. sys.stdout.write("\033[F")
  88. current_file.write('\n')
  89. print('\n')
  90. # save current information about min file found
  91. if mode == 'svd':
  92. with open(data_min_max_filename, 'w') as f:
  93. f.write(str(min_val_found) + '\n')
  94. f.write(str(max_val_found) + '\n')
  95. print("%s_%s : end of data generation\n" % (data_type, mode))
  96. def main():
  97. parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes (keep in memory min and max value found)")
  98. parser.add_argument('--feature', type=str,
  99. help="feature choice in order to compute data (use 'all' if all features are needed)")
  100. parser.add_argument('--folder', type=str, help="folder which contains the whole dataset")
  101. args = parser.parse_args()
  102. p_feature = args.feature
  103. p_folder = args.folder
  104. # generate all or specific feature data
  105. if p_feature == 'all':
  106. for m in features_choices:
  107. generate_data_svd(m, 'svd', p_folder)
  108. generate_data_svd(m, 'svdn', p_folder)
  109. generate_data_svd(m, 'svdne', p_folder)
  110. else:
  111. if p_feature not in features_choices:
  112. raise ValueError('Unknown feature choice : ', features_choices)
  113. generate_data_svd(p_feature, 'svd', p_folder)
  114. generate_data_svd(p_feature, 'svdn', p_folder)
  115. generate_data_svd(p_feature, 'svdne', p_folder)
  116. if __name__== "__main__":
  117. main()