save_model_result_in_md_maxwell.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. # main imports
  2. import numpy as np
  3. import pandas as pd
  4. import sys, os, argparse
  5. import subprocess
  6. import time
  7. import json
  8. # models imports
  9. from sklearn.utils import shuffle
  10. from sklearn.externals import joblib
  11. from sklearn.metrics import accuracy_score, f1_score, recall_score, roc_auc_score
  12. from sklearn.model_selection import cross_val_score
  13. from sklearn.model_selection import StratifiedKFold
  14. from sklearn.model_selection import train_test_split
  15. from keras.models import Sequential
  16. from keras.layers import Conv1D, MaxPooling1D
  17. from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
  18. from keras.wrappers.scikit_learn import KerasClassifier
  19. from keras import backend as K
  20. from keras.models import model_from_json
  21. # image processing imports
  22. from ipfml import processing
  23. from PIL import Image
  24. # modules imports
  25. sys.path.insert(0, '') # trick to enable import of main folder module
  26. import custom_config as cfg
  27. # variables and parameters
  28. threshold_map_folder = cfg.threshold_map_folder
  29. threshold_map_file_prefix = cfg.threshold_map_folder + "_"
  30. markdowns_folder = cfg.models_information_folder
  31. final_csv_model_comparisons = cfg.csv_model_comparisons_filename
  32. models_name = cfg.models_names_list
  33. zones = cfg.zones_indices
  34. current_dirpath = os.getcwd()
  35. def main():
  36. kind_model = 'keras'
  37. model_ext = ''
  38. parser = argparse.ArgumentParser(description="Display SVD data of scene zone")
  39. parser.add_argument('--interval', type=str, help='Interval value to keep from svd', default='"0, 200"')
  40. parser.add_argument('--model', type=str, help='.joblib or .json file (sklearn or keras model)')
  41. parser.add_argument('--metric', type=str, help='Metric data choice', choices=cfg.metric_choices_labels)
  42. parser.add_argument('--mode', type=str, help='Kind of normalization level wished', choices=cfg.normalization_choices)
  43. args = parser.parse_args()
  44. p_interval = list(map(int, args.interval.split(',')))
  45. p_model_file = args.model
  46. p_metric = args.metric
  47. p_mode = args.mode
  48. # call model and get global result in scenes
  49. begin, end = p_interval
  50. bash_cmd = "bash others/testModelByScene_maxwell.sh '" + str(begin) + "' '" + str(end) + "' '" + p_model_file + "' '" + p_mode + "' '" + p_metric + "'"
  51. print(bash_cmd)
  52. ## call command ##
  53. p = subprocess.Popen(bash_cmd, stdout=subprocess.PIPE, shell=True)
  54. (output, err) = p.communicate()
  55. ## Wait for result ##
  56. p_status = p.wait()
  57. if not os.path.exists(markdowns_folder):
  58. os.makedirs(markdowns_folder)
  59. # get model name to construct model
  60. if '.joblib' in p_model_file:
  61. kind_model = 'sklearn'
  62. model_ext = '.joblib'
  63. if '.json' in p_model_file:
  64. kind_model = 'keras'
  65. model_ext = '.json'
  66. md_model_path = os.path.join(markdowns_folder, p_model_file.split('/')[-1].replace(model_ext, '.md'))
  67. with open(md_model_path, 'w') as f:
  68. f.write(output.decode("utf-8"))
  69. # read each threshold_map information if exists
  70. model_map_info_path = os.path.join(threshold_map_folder, p_model_file.replace('saved_models/', ''))
  71. if not os.path.exists(model_map_info_path):
  72. f.write('\n\n No threshold map information')
  73. else:
  74. maps_files = os.listdir(model_map_info_path)
  75. # get all map information
  76. for t_map_file in maps_files:
  77. file_path = os.path.join(model_map_info_path, t_map_file)
  78. with open(file_path, 'r') as map_file:
  79. title_scene = t_map_file.replace(threshold_map_file_prefix, '')
  80. f.write('\n\n## ' + title_scene + '\n')
  81. content = map_file.readlines()
  82. # getting each map line information
  83. for line in content:
  84. f.write(line)
  85. f.close()
  86. # Keep model information to compare
  87. current_model_name = p_model_file.split('/')[-1].replace(model_ext, '')
  88. # Prepare writing in .csv file into results folder
  89. output_final_file_path = os.path.join(cfg.results_information_folder, final_csv_model_comparisons)
  90. output_final_file = open(output_final_file_path, "a")
  91. print(current_model_name)
  92. # reconstruct data filename
  93. for name in models_name:
  94. if name in current_model_name:
  95. data_filename = current_model_name
  96. current_data_file_path = os.path.join('data', data_filename)
  97. print("Current data file ")
  98. print(current_data_file_path)
  99. model_scores = []
  100. ########################
  101. # 1. Get and prepare data
  102. ########################
  103. dataset_train = pd.read_csv(current_data_file_path + '.train', header=None, sep=";")
  104. dataset_test = pd.read_csv(current_data_file_path + '.test', header=None, sep=";")
  105. # default first shuffle of data
  106. dataset_train = shuffle(dataset_train)
  107. dataset_test = shuffle(dataset_test)
  108. # get dataset with equal number of classes occurences
  109. noisy_df_train = dataset_train[dataset_train.ix[:, 0] == 1]
  110. not_noisy_df_train = dataset_train[dataset_train.ix[:, 0] == 0]
  111. nb_noisy_train = len(noisy_df_train.index)
  112. noisy_df_test = dataset_test[dataset_test.ix[:, 0] == 1]
  113. not_noisy_df_test = dataset_test[dataset_test.ix[:, 0] == 0]
  114. nb_noisy_test = len(noisy_df_test.index)
  115. final_df_train = pd.concat([not_noisy_df_train[0:nb_noisy_train], noisy_df_train])
  116. final_df_test = pd.concat([not_noisy_df_test[0:nb_noisy_test], noisy_df_test])
  117. # shuffle data another time
  118. final_df_train = shuffle(final_df_train)
  119. final_df_test = shuffle(final_df_test)
  120. final_df_train_size = len(final_df_train.index)
  121. final_df_test_size = len(final_df_test.index)
  122. # use of the whole data set for training
  123. x_dataset_train = final_df_train.ix[:,1:]
  124. x_dataset_test = final_df_test.ix[:,1:]
  125. y_dataset_train = final_df_train.ix[:,0]
  126. y_dataset_test = final_df_test.ix[:,0]
  127. #######################
  128. # 2. Getting model
  129. #######################
  130. if kind_model == 'keras':
  131. with open(p_model_file, 'r') as f:
  132. json_model = json.load(f)
  133. model = model_from_json(json_model)
  134. model.load_weights(p_model_file.replace('.json', '.h5'))
  135. model.compile(loss='binary_crossentropy',
  136. optimizer='adam',
  137. metrics=['accuracy'])
  138. # reshape all input data
  139. x_dataset_train = np.array(x_dataset_train).reshape(len(x_dataset_train), end, 1)
  140. x_dataset_test = np.array(x_dataset_test).reshape(len(x_dataset_test), end, 1)
  141. if kind_model == 'sklearn':
  142. model = joblib.load(p_model_file)
  143. #######################
  144. # 3. Fit model : use of cross validation to fit model
  145. #######################
  146. if kind_model == 'keras':
  147. model.fit(x_dataset_train, y_dataset_train, validation_split=0.20, epochs=cfg.keras_epochs, batch_size=cfg.keras_batch)
  148. if kind_model == 'sklearn':
  149. model.fit(x_dataset_train, y_dataset_train)
  150. train_accuracy = cross_val_score(model, x_dataset_train, y_dataset_train, cv=5)
  151. ######################
  152. # 4. Test : Validation and test dataset from .test dataset
  153. ######################
  154. # we need to specify validation size to 20% of whole dataset
  155. val_set_size = int(final_df_train_size/3)
  156. test_set_size = val_set_size
  157. total_validation_size = val_set_size + test_set_size
  158. if final_df_test_size > total_validation_size:
  159. x_dataset_test = x_dataset_test[0:total_validation_size]
  160. y_dataset_test = y_dataset_test[0:total_validation_size]
  161. X_test, X_val, y_test, y_val = train_test_split(x_dataset_test, y_dataset_test, test_size=0.5, random_state=1)
  162. if kind_model == 'keras':
  163. y_test_model = model.predict_classes(X_test)
  164. y_val_model = model.predict_classes(X_val)
  165. y_train_model = model.predict_classes(x_dataset_train)
  166. train_accuracy = accuracy_score(y_dataset_train, y_train_model)
  167. if kind_model == 'sklearn':
  168. y_test_model = model.predict(X_test)
  169. y_val_model = model.predict(X_val)
  170. y_train_model = model.predict(x_dataset_train)
  171. val_accuracy = accuracy_score(y_val, y_val_model)
  172. test_accuracy = accuracy_score(y_test, y_test_model)
  173. train_f1 = f1_score(y_dataset_train, y_train_model)
  174. train_recall = recall_score(y_dataset_train, y_train_model)
  175. train_roc_auc = roc_auc_score(y_dataset_train, y_train_model)
  176. val_f1 = f1_score(y_val, y_val_model)
  177. val_recall = recall_score(y_val, y_val_model)
  178. val_roc_auc = roc_auc_score(y_val, y_val_model)
  179. test_f1 = f1_score(y_test, y_test_model)
  180. test_recall = recall_score(y_test, y_test_model)
  181. test_roc_auc = roc_auc_score(y_test, y_test_model)
  182. if kind_model == 'keras':
  183. # stats of all dataset
  184. all_x_data = np.concatenate([x_dataset_train, X_test, X_val])
  185. all_y_data = np.concatenate([y_dataset_train, y_test, y_val])
  186. all_y_model = model.predict_classes(all_x_data)
  187. if kind_model == 'sklearn':
  188. # stats of all dataset
  189. all_x_data = pd.concat([x_dataset_train, X_test, X_val])
  190. all_y_data = pd.concat([y_dataset_train, y_test, y_val])
  191. all_y_model = model.predict(all_x_data)
  192. all_accuracy = accuracy_score(all_y_data, all_y_model)
  193. all_f1_score = f1_score(all_y_data, all_y_model)
  194. all_recall_score = recall_score(all_y_data, all_y_model)
  195. all_roc_auc_score = roc_auc_score(all_y_data, all_y_model)
  196. # stats of dataset sizes
  197. total_samples = final_df_train_size + val_set_size + test_set_size
  198. model_scores.append(final_df_train_size)
  199. model_scores.append(val_set_size)
  200. model_scores.append(test_set_size)
  201. model_scores.append(final_df_train_size / total_samples)
  202. model_scores.append(val_set_size / total_samples)
  203. model_scores.append(test_set_size / total_samples)
  204. # add of scores
  205. model_scores.append(train_accuracy)
  206. model_scores.append(val_accuracy)
  207. model_scores.append(test_accuracy)
  208. model_scores.append(all_accuracy)
  209. model_scores.append(train_f1)
  210. model_scores.append(train_recall)
  211. model_scores.append(train_roc_auc)
  212. model_scores.append(val_f1)
  213. model_scores.append(val_recall)
  214. model_scores.append(val_roc_auc)
  215. model_scores.append(test_f1)
  216. model_scores.append(test_recall)
  217. model_scores.append(test_roc_auc)
  218. model_scores.append(all_f1_score)
  219. model_scores.append(all_recall_score)
  220. model_scores.append(all_roc_auc_score)
  221. # TODO : improve...
  222. # check if it's always the case...
  223. nb_zones = current_data_file_path.split('_')[7]
  224. final_file_line = current_model_name + '; ' + str(end - begin) + '; ' + str(begin) + '; ' + str(end) + '; ' + str(nb_zones) + '; ' + p_metric + '; ' + p_mode
  225. for s in model_scores:
  226. final_file_line += '; ' + str(s)
  227. output_final_file.write(final_file_line + '\n')
  228. if __name__== "__main__":
  229. main()