train_lstm_weighted.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373
  1. # main imports
  2. import argparse, sys
  3. import numpy as np
  4. import pandas as pd
  5. import os
  6. import ctypes
  7. from PIL import Image
  8. import cv2
  9. from keras import backend as K
  10. import matplotlib.pyplot as plt
  11. from ipfml import utils
  12. # dl imports
  13. from keras.layers import Dense, Dropout, LSTM, Embedding, GRU, BatchNormalization, ConvLSTM2D, Conv3D, Flatten
  14. from keras.preprocessing.sequence import pad_sequences
  15. from keras.models import Sequential
  16. from keras.models import load_model
  17. from keras.callbacks import ModelCheckpoint
  18. from sklearn.metrics import roc_auc_score, accuracy_score
  19. import tensorflow as tf
  20. from keras import backend as K
  21. import sklearn
  22. from sklearn.model_selection import train_test_split
  23. from joblib import dump
  24. import custom_config as cfg
  25. # global variables
  26. n_counter = 0
  27. total_samples = 0
  28. def write_progress(progress):
  29. '''
  30. Display progress information as progress bar
  31. '''
  32. barWidth = 180
  33. output_str = "["
  34. pos = barWidth * progress
  35. for i in range(barWidth):
  36. if i < pos:
  37. output_str = output_str + "="
  38. elif i == pos:
  39. output_str = output_str + ">"
  40. else:
  41. output_str = output_str + " "
  42. output_str = output_str + "] " + str(int(progress * 100.0)) + " %\r"
  43. print(output_str)
  44. sys.stdout.write("\033[F")
  45. def build_input(df, seq_norm, p_chanels):
  46. """Convert dataframe to numpy array input with timesteps as float array
  47. Arguments:
  48. df: {pd.Dataframe} -- Dataframe input
  49. seq_norm: {bool} -- normalize or not seq input data by features
  50. Returns:
  51. {np.ndarray} -- input LSTM data as numpy array
  52. """
  53. global n_counter
  54. global total_samples
  55. arr = []
  56. # for each input line
  57. for row in df.iterrows():
  58. seq_arr = []
  59. # for each sequence data input
  60. for column in row[1]:
  61. seq_elems = []
  62. # for each element in sequence data
  63. for i, img_path in enumerate(column):
  64. # seq_elems.append(np.array(img).flatten())
  65. if p_chanels[i] > 1:
  66. img = cv2.imread(img_path)
  67. else:
  68. img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
  69. seq_elems.append(np.array(img, 'float32') / 255.)
  70. #seq_arr.append(np.array(seq_elems).flatten())
  71. seq_arr.append(np.array(seq_elems))
  72. arr.append(seq_arr)
  73. # update progress
  74. n_counter += 1
  75. write_progress(n_counter / float(total_samples))
  76. arr = np.array(arr)
  77. print(arr.shape)
  78. # final_arr = []
  79. # for v in arr:
  80. # v_data = []
  81. # for vv in v:
  82. # #scaled_vv = np.array(vv, 'float') - np.mean(np.array(vv, 'float'))
  83. # #v_data.append(scaled_vv)
  84. # v_data.append(vv)
  85. # final_arr.append(v_data)
  86. final_arr = np.array(arr, 'float32')
  87. # check if sequence normalization is used
  88. if seq_norm:
  89. if final_arr.ndim > 2:
  90. n, s, f = final_arr.shape
  91. for index, seq in enumerate(final_arr):
  92. for i in range(f):
  93. final_arr[index][:, i] = utils.normalize_arr_with_range(seq[:, i])
  94. return final_arr
  95. def create_model(_input_shape):
  96. print ('Creating model...')
  97. model = Sequential()
  98. # model.add(Conv3D(60, (1, 2, 2), input_shape=input_shape))
  99. # model.add(Activation('relu'))
  100. # model.add(MaxPooling3D(pool_size=(1, 2, 2)))
  101. #model.add(Embedding(input_dim = 1000, output_dim = 50, input_length=input_length))
  102. # model.add(ConvLSTM2D(filters=40, kernel_size=(3, 3), input_shape=input_shape, units=256, activation='sigmoid', recurrent_activation='hard_sigmoid'))
  103. # model.add(Dropout(0.4))
  104. # model.add(GRU(units=128, activation='sigmoid', recurrent_activation='hard_sigmoid'))
  105. # model.add(Dropout(0.4))
  106. # model.add(Dense(1, activation='sigmoid'))
  107. model.add(ConvLSTM2D(filters=100, kernel_size=(3, 3),
  108. input_shape=_input_shape,
  109. dropout=0.5,
  110. #recurrent_dropout=0.5,
  111. padding='same', return_sequences=True))
  112. model.add(BatchNormalization())
  113. model.add(ConvLSTM2D(filters=30, kernel_size=(3, 3),
  114. dropout=0.5,
  115. #recurrent_dropout=0.5,
  116. padding='same', return_sequences=True))
  117. model.add(BatchNormalization())
  118. model.add(Dropout(0.5))
  119. model.add(Conv3D(filters=15, kernel_size=(3, 3, 3),
  120. activation='sigmoid',
  121. padding='same', data_format='channels_last'))
  122. model.add(Dropout(0.5))
  123. model.add(Flatten())
  124. model.add(Dense(512, activation='relu'))
  125. model.add(BatchNormalization())
  126. model.add(Dropout(0.5))
  127. model.add(Dense(128, activation='relu'))
  128. model.add(BatchNormalization())
  129. model.add(Dropout(0.5))
  130. model.add(Dense(1, activation='sigmoid'))
  131. model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
  132. print ('-- Compiling...')
  133. return model
  134. def main():
  135. # get this variable as global
  136. global total_samples
  137. parser = argparse.ArgumentParser(description="Read and compute training of LSTM model")
  138. parser.add_argument('--train', type=str, help='input train dataset', required=True)
  139. parser.add_argument('--test', type=str, help='input test dataset', required=True)
  140. parser.add_argument('--output', type=str, help='output model name', required=True)
  141. parser.add_argument('--chanels', type=str, help="given number of ordered chanels (example: '1,3,3') for each element of window", required=True)
  142. parser.add_argument('--epochs', type=int, help='number of expected epochs', default=30)
  143. parser.add_argument('--batch_size', type=int, help='expected batch size for training model', default=64)
  144. parser.add_argument('--seq_norm', type=int, help='normalization sequence by features', choices=[0, 1], default=0)
  145. args = parser.parse_args()
  146. p_train = args.train
  147. p_test = args.test
  148. p_output = args.output
  149. p_chanels = list(map(int, args.chanels.split(',')))
  150. p_epochs = args.epochs
  151. p_batch_size = args.batch_size
  152. p_seq_norm = bool(args.seq_norm)
  153. print('-----------------------------')
  154. print("----- Preparing data... -----")
  155. dataset_train = pd.read_csv(p_train, header=None, sep=';')
  156. dataset_test = pd.read_csv(p_test, header=None, sep=';')
  157. print("-- Train set size : ", len(dataset_train))
  158. print("-- Test set size : ", len(dataset_test))
  159. # getting weighted class over the whole dataset
  160. noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 1]
  161. not_noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 0]
  162. nb_noisy_train = len(noisy_df_train.index)
  163. nb_not_noisy_train = len(not_noisy_df_train.index)
  164. noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 1]
  165. not_noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 0]
  166. nb_noisy_test = len(noisy_df_test.index)
  167. nb_not_noisy_test = len(not_noisy_df_test.index)
  168. noisy_samples = nb_noisy_test + nb_noisy_train
  169. not_noisy_samples = nb_not_noisy_test + nb_not_noisy_train
  170. total_samples = noisy_samples + not_noisy_samples
  171. print('-----------------------------')
  172. print('---- Dataset information ----')
  173. print('-- noisy:', noisy_samples)
  174. print('-- not_noisy:', not_noisy_samples)
  175. print('-- total:', total_samples)
  176. print('-----------------------------')
  177. class_weight = {
  178. 0: noisy_samples / float(total_samples),
  179. 1: (not_noisy_samples / float(total_samples)),
  180. }
  181. # shuffle data
  182. final_df_train = sklearn.utils.shuffle(dataset_train)
  183. final_df_test = sklearn.utils.shuffle(dataset_test)
  184. print('---- Loading dataset.... ----')
  185. print('-----------------------------\n')
  186. # split dataset into X_train, y_train, X_test, y_test
  187. X_train_all = final_df_train.loc[:, 1:].apply(lambda x: x.astype(str).str.split('::'))
  188. X_train_all = build_input(X_train_all, p_seq_norm, p_chanels)
  189. y_train_all = final_df_train.loc[:, 0].astype('int')
  190. X_test = final_df_test.loc[:, 1:].apply(lambda x: x.astype(str).str.split('::'))
  191. X_test = build_input(X_test, p_seq_norm, p_chanels)
  192. y_test = final_df_test.loc[:, 0].astype('int')
  193. input_shape = (X_train_all.shape[1], X_train_all.shape[2], X_train_all.shape[3], X_train_all.shape[4])
  194. print('\n-----------------------------')
  195. print('-- Training data input shape', input_shape)
  196. print('-----------------------------')
  197. # create backup folder for current model
  198. model_backup_folder = os.path.join(cfg.backup_model_folder, p_output)
  199. if not os.path.exists(model_backup_folder):
  200. os.makedirs(model_backup_folder)
  201. # add of callback models
  202. filepath = os.path.join(cfg.backup_model_folder, p_output, p_output + "-_{epoch:03d}.h5")
  203. checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=0, mode='max')
  204. callbacks_list = [checkpoint]
  205. # check if backup already exists
  206. backups = sorted(os.listdir(model_backup_folder))
  207. if len(backups) > 0:
  208. last_backup_file = backups[-1]
  209. model = load_model(last_backup_file)
  210. # get initial epoch
  211. initial_epoch = int(last_backup_file.split('_')[-1].replace('.h5', ''))
  212. print('-----------------------------')
  213. print('-- Restore model from backup...')
  214. print('-- Restart training @epoch:', initial_epoch)
  215. print('-----------------------------')
  216. else:
  217. model = create_model(input_shape)
  218. model.summary()
  219. # prepare train and validation dataset
  220. X_train, X_val, y_train, y_val = train_test_split(X_train_all, y_train_all, test_size=0.3, shuffle=False)
  221. print("-- Fitting model with custom class_weight", class_weight)
  222. print('-----------------------------')
  223. history = model.fit(X_train, y_train, batch_size=p_batch_size, epochs=p_epochs, validation_data=(X_val, y_val), verbose=1, shuffle=True, class_weight=class_weight)
  224. # list all data in history
  225. # print(history.history.keys())
  226. # # summarize history for accuracy
  227. # plt.plot(history.history['accuracy'])
  228. # plt.plot(history.history['val_accuracy'])
  229. # plt.title('model accuracy')
  230. # plt.ylabel('accuracy')
  231. # plt.xlabel('epoch')
  232. # plt.legend(['train', 'test'], loc='upper left')
  233. # plt.show()
  234. # # summarize history for loss
  235. # plt.plot(history.history['loss'])
  236. # plt.plot(history.history['val_loss'])
  237. # plt.title('model loss')
  238. # plt.ylabel('loss')
  239. # plt.xlabel('epoch')
  240. # plt.legend(['train', 'test'], loc='upper left')
  241. # plt.show()
  242. # train_score, train_acc = model.evaluate(X_train, y_train, batch_size=1)
  243. # print(train_acc)
  244. y_train_predict = model.predict(X_train, batch_size=1, verbose=1)
  245. y_val_predict = model.predict(X_val, batch_size=1, verbose=1)
  246. y_test_predict = model.predict(X_test, batch_size=1, verbose=1)
  247. y_train_predict = [ 1 if l > 0.5 else 0 for l in y_train_predict ]
  248. y_val_predict = [ 1 if l > 0.5 else 0 for l in y_val_predict ]
  249. y_test_predict = [ 1 if l > 0.5 else 0 for l in y_test_predict ]
  250. auc_train = roc_auc_score(y_train, y_train_predict)
  251. auc_val = roc_auc_score(y_val, y_val_predict)
  252. auc_test = roc_auc_score(y_test, y_test_predict)
  253. acc_train = accuracy_score(y_train, y_train_predict)
  254. acc_val = accuracy_score(y_val, y_val_predict)
  255. acc_test = accuracy_score(y_test, y_test_predict)
  256. print('Train ACC:', acc_train)
  257. print('Train AUC', auc_train)
  258. print('Val ACC:', acc_val)
  259. print('Val AUC', auc_val)
  260. print('Test ACC:', acc_test)
  261. print('Test AUC:', auc_test)
  262. # save acc metric information
  263. plt.plot(history.history['accuracy'])
  264. plt.plot(history.history['val_accuracy'])
  265. plt.title('model accuracy')
  266. plt.ylabel('accuracy')
  267. plt.xlabel('epoch')
  268. plt.legend(['train', 'test'], loc='upper left')
  269. model_history = os.path.join(cfg.output_results_folder, p_output + '.png')
  270. plt.savefig(model_history)
  271. # save model using keras API
  272. if not os.path.exists(cfg.output_models):
  273. os.makedirs(cfg.output_models)
  274. model.save(os.path.join(cfg.output_models, p_output + '.h5'))
  275. # save model results
  276. if not os.path.exists(cfg.output_results_folder):
  277. os.makedirs(cfg.output_results_folder)
  278. results_filename_path = os.path.join(cfg.output_results_folder, cfg.results_filename)
  279. if not os.path.exists(results_filename_path):
  280. with open(results_filename_path, 'w') as f:
  281. f.write('name;train_acc;val_acc;test_acc;train_auc;val_auc;test_auc;\n')
  282. with open(results_filename_path, 'a') as f:
  283. f.write(p_output + ';' + str(acc_train) + ';' + str(acc_val) + ';' + str(acc_test) + ';' \
  284. + str(auc_train) + ';' + str(auc_val) + ';' + str(auc_test) + '\n')
  285. if __name__ == "__main__":
  286. main()