save_model_result_in_md_maxwell.py 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236
  1. from sklearn.utils import shuffle
  2. from sklearn.externals import joblib
  3. from sklearn.metrics import accuracy_score, f1_score
  4. from sklearn.model_selection import cross_val_score
  5. from sklearn.model_selection import train_test_split
  6. import numpy as np
  7. import pandas as pd
  8. from ipfml import image_processing
  9. from PIL import Image
  10. import sys, os, getopt
  11. import subprocess
  12. import time
  13. current_dirpath = os.getcwd()
  14. threshold_map_folder = "threshold_map"
  15. threshold_map_file_prefix = "treshold_map_"
  16. markdowns_folder = "models_info"
  17. final_csv_model_comparisons = "models_comparisons.csv"
  18. models_name = ["svm_model","ensemble_model","ensemble_model_v2"]
  19. zones = np.arange(16)
  20. def main():
  21. if len(sys.argv) <= 1:
  22. print('Run with default parameters...')
  23. print('python save_model_result_in_md.py --interval "0,20" --model path/to/xxxx.joblib --mode ["svd", "svdn", "svdne"] --metric ["lab", "mscn"]')
  24. sys.exit(2)
  25. try:
  26. opts, args = getopt.getopt(sys.argv[1:], "ht:m:o:l", ["help=", "interval=", "model=", "mode=", "metric="])
  27. except getopt.GetoptError:
  28. # print help information and exit:
  29. print('python save_model_result_in_md.py --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svd", "svdn", "svdne"] --metric ["lab", "mscn"]')
  30. sys.exit(2)
  31. for o, a in opts:
  32. if o == "-h":
  33. print('python save_model_result_in_md.py --interval "xx,xx" --model path/to/xxxx.joblib --mode ["svd", "svdn", "svdne"] --metric ["lab", "mscn"]')
  34. sys.exit()
  35. elif o in ("-t", "--interval"):
  36. p_interval = list(map(int, a.split(',')))
  37. elif o in ("-m", "--model"):
  38. p_model_file = a
  39. elif o in ("-o", "--mode"):
  40. p_mode = a
  41. if p_mode != 'svdn' and p_mode != 'svdne' and p_mode != 'svd':
  42. assert False, "Mode not recognized"
  43. elif o in ("-c", "--metric"):
  44. p_metric = a
  45. else:
  46. assert False, "unhandled option"
  47. # call model and get global result in scenes
  48. begin, end = p_interval
  49. bash_cmd = "bash testModelByScene_maxwell.sh '" + str(begin) + "' '" + str(end) + "' '" + p_model_file + "' '" + p_mode + "' '" + p_metric + "'"
  50. print(bash_cmd)
  51. ## call command ##
  52. p = subprocess.Popen(bash_cmd, stdout=subprocess.PIPE, shell=True)
  53. (output, err) = p.communicate()
  54. ## Wait for result ##
  55. p_status = p.wait()
  56. if not os.path.exists(markdowns_folder):
  57. os.makedirs(markdowns_folder)
  58. # get model name to construct model
  59. md_model_path = os.path.join(markdowns_folder, p_model_file.split('/')[-1].replace('.joblib', '.md'))
  60. with open(md_model_path, 'w') as f:
  61. f.write(output.decode("utf-8"))
  62. # read each threshold_map information if exists
  63. model_map_info_path = os.path.join(threshold_map_folder, p_model_file.replace('saved_models/', ''))
  64. if not os.path.exists(model_map_info_path):
  65. f.write('\n\n No threshold map information')
  66. else:
  67. maps_files = os.listdir(model_map_info_path)
  68. # get all map information
  69. for t_map_file in maps_files:
  70. file_path = os.path.join(model_map_info_path, t_map_file)
  71. with open(file_path, 'r') as map_file:
  72. title_scene = t_map_file.replace(threshold_map_file_prefix, '')
  73. f.write('\n\n## ' + title_scene + '\n')
  74. content = map_file.readlines()
  75. # getting each map line information
  76. for line in content:
  77. f.write(line)
  78. f.close()
  79. # Keep model information to compare
  80. current_model_name = p_model_file.split('/')[-1].replace('.joblib', '')
  81. # Prepare writing in .csv file
  82. output_final_file_path = os.path.join(markdowns_folder, final_csv_model_comparisons)
  83. output_final_file = open(output_final_file_path, "a")
  84. print(current_model_name)
  85. # reconstruct data filename
  86. for name in models_name:
  87. if name in current_model_name:
  88. current_data_file_path = os.path.join('data', current_model_name.replace(name, 'data_maxwell'))
  89. model_scores = []
  90. ########################
  91. # 1. Get and prepare data
  92. ########################
  93. dataset_train = pd.read_csv(current_data_file_path + '.train', header=None, sep=";")
  94. dataset_test = pd.read_csv(current_data_file_path + '.test', header=None, sep=";")
  95. # default first shuffle of data
  96. dataset_train = shuffle(dataset_train)
  97. dataset_test = shuffle(dataset_test)
  98. # get dataset with equal number of classes occurences
  99. noisy_df_train = dataset_train[dataset_train.ix[:, 0] == 1]
  100. not_noisy_df_train = dataset_train[dataset_train.ix[:, 0] == 0]
  101. nb_noisy_train = len(noisy_df_train.index)
  102. noisy_df_test = dataset_test[dataset_test.ix[:, 0] == 1]
  103. not_noisy_df_test = dataset_test[dataset_test.ix[:, 0] == 0]
  104. nb_noisy_test = len(noisy_df_test.index)
  105. final_df_train = pd.concat([not_noisy_df_train[0:nb_noisy_train], noisy_df_train])
  106. final_df_test = pd.concat([not_noisy_df_test[0:nb_noisy_test], noisy_df_test])
  107. # shuffle data another time
  108. final_df_train = shuffle(final_df_train)
  109. final_df_test = shuffle(final_df_test)
  110. final_df_train_size = len(final_df_train.index)
  111. final_df_test_size = len(final_df_test.index)
  112. # use of the whole data set for training
  113. x_dataset_train = final_df_train.ix[:,1:]
  114. x_dataset_test = final_df_test.ix[:,1:]
  115. y_dataset_train = final_df_train.ix[:,0]
  116. y_dataset_test = final_df_test.ix[:,0]
  117. #######################
  118. # 2. Getting model
  119. #######################
  120. model = joblib.load(p_model_file)
  121. #######################
  122. # 3. Fit model : use of cross validation to fit model
  123. #######################
  124. model.fit(x_dataset_train, y_dataset_train)
  125. val_scores = cross_val_score(model, x_dataset_train, y_dataset_train, cv=5)
  126. ######################
  127. # 4. Test : Validation and test dataset from .test dataset
  128. ######################
  129. # we need to specify validation size to 20% of whole dataset
  130. val_set_size = int(final_df_train_size/3)
  131. test_set_size = val_set_size
  132. total_validation_size = val_set_size + test_set_size
  133. if final_df_test_size > total_validation_size:
  134. x_dataset_test = x_dataset_test[0:total_validation_size]
  135. y_dataset_test = y_dataset_test[0:total_validation_size]
  136. X_test, X_val, y_test, y_val = train_test_split(x_dataset_test, y_dataset_test, test_size=0.5, random_state=1)
  137. y_test_model = model.predict(X_test)
  138. y_val_model = model.predict(X_val)
  139. val_accuracy = accuracy_score(y_val, y_val_model)
  140. test_accuracy = accuracy_score(y_test, y_test_model)
  141. y_train_model = model.predict(x_dataset_train)
  142. train_f1 = f1_score(y_dataset_train, y_train_model)
  143. val_f1 = f1_score(y_val, y_val_model)
  144. test_f1 = f1_score(y_test, y_test_model)
  145. # stats of all dataset
  146. all_x_data = pd.concat([x_dataset_train, X_test, X_val])
  147. all_y_data = pd.concat([y_dataset_train, y_test, y_val])
  148. all_y_model = model.predict(all_x_data)
  149. all_accuracy = accuracy_score(all_y_data, all_y_model)
  150. all_f1_score = f1_score(all_y_data, all_y_model)
  151. # stats of dataset sizes
  152. total_samples = final_df_train_size + val_set_size + test_set_size
  153. model_scores.append(final_df_train_size / total_samples)
  154. model_scores.append(val_set_size / total_samples)
  155. model_scores.append(test_set_size / total_samples)
  156. # add of scores
  157. model_scores.append(val_scores.mean())
  158. model_scores.append(val_accuracy)
  159. model_scores.append(test_accuracy)
  160. model_scores.append(all_accuracy)
  161. model_scores.append(train_f1)
  162. model_scores.append(val_f1)
  163. model_scores.append(test_f1)
  164. model_scores.append(all_f1_score)
  165. # TODO : improve...
  166. # check if it's always the case...
  167. nb_zones = current_data_file_path.split('_')[7]
  168. final_file_line = current_model_name + '; ' + str(end - begin) + '; ' + str(begin) + '; ' + str(end) + '; ' + str(nb_zones) + '; ' + p_metric + '; ' + p_mode
  169. for s in model_scores:
  170. final_file_line += '; ' + str(s)
  171. output_final_file.write(final_file_line + '\n')
  172. if __name__== "__main__":
  173. main()