find_best_attributes_surrogate_openML_multi_specific.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. # main imports
  2. import os
  3. import sys
  4. import argparse
  5. import pandas as pd
  6. import numpy as np
  7. import logging
  8. import datetime
  9. import random
  10. # model imports
  11. from sklearn.model_selection import train_test_split
  12. from sklearn.model_selection import GridSearchCV
  13. from sklearn.linear_model import LogisticRegression
  14. from sklearn.ensemble import RandomForestClassifier, VotingClassifier
  15. import joblib
  16. import sklearn
  17. import sklearn.svm as svm
  18. from sklearn.utils import shuffle
  19. from sklearn.metrics import roc_auc_score
  20. from sklearn.model_selection import cross_val_score
  21. from sklearn.preprocessing import MinMaxScaler
  22. # modules and config imports
  23. sys.path.insert(0, '') # trick to enable import of main folder module
  24. import custom_config as cfg
  25. import models as mdl
  26. from optimization.ILSMultiSpecificSurrogate import ILSMultiSpecificSurrogate
  27. from macop.solutions.BinarySolution import BinarySolution
  28. from macop.operators.mutators.SimpleMutation import SimpleMutation
  29. from macop.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
  30. from macop.operators.crossovers.SimpleCrossover import SimpleCrossover
  31. from macop.operators.crossovers.RandomSplitCrossover import RandomSplitCrossover
  32. from macop.operators.policies.UCBPolicy import UCBPolicy
  33. from macop.operators.policies.RandomPolicy import RandomPolicy
  34. from macop.callbacks.BasicCheckpoint import BasicCheckpoint
  35. from macop.callbacks.UCBCheckpoint import UCBCheckpoint
  36. from optimization.callbacks.SurrogateCheckpoint import SurrogateCheckpoint
  37. from optimization.callbacks.MultiSurrogateCheckpoint import MultiSurrogateCheckpoint
  38. from sklearn.ensemble import RandomForestClassifier
  39. # avoid display of warning
  40. def warn(*args, **kwargs):
  41. pass
  42. import warnings
  43. warnings.filterwarnings("ignore", category=DeprecationWarning)
  44. warnings.warn = warn
  45. # default validator
  46. def validator(solution):
  47. # at least 5 attributes
  48. if list(solution._data).count(1) < 2:
  49. return False
  50. return True
  51. def train_model(X_train, y_train):
  52. #print ('Creating model...')
  53. # here use of SVM with grid search CV
  54. Cs = [0.001, 0.01, 0.1, 1, 10, 100]
  55. gammas = [0.001, 0.01, 0.1,10, 100]
  56. param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
  57. svc = svm.SVC(probability=True, class_weight='balanced')
  58. #clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
  59. clf = GridSearchCV(svc, param_grid, cv=4, verbose=0, n_jobs=-1)
  60. clf.fit(X_train, y_train)
  61. model = clf.best_estimator_
  62. return model
  63. def loadDataset(filename):
  64. ########################
  65. # 1. Get and prepare data
  66. ########################
  67. dataset = pd.read_csv(filename, sep=',')
  68. # change label as common
  69. min_label_value = min(dataset.iloc[:, -1])
  70. max_label_value = max(dataset.iloc[:, -1])
  71. dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(min_label_value, 0)
  72. dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(max_label_value, 1)
  73. X_dataset = dataset.iloc[:, :-1]
  74. y_dataset = dataset.iloc[:, -1]
  75. problem_size = len(X_dataset.columns)
  76. # min/max normalisation over feature
  77. # create a scaler object
  78. scaler = MinMaxScaler()
  79. # fit and transform the data
  80. X_dataset = np.array(pd.DataFrame(scaler.fit_transform(X_dataset), columns=X_dataset.columns))
  81. # prepare train, validation and test datasets
  82. X_train, X_test, y_train, y_test = train_test_split(X_dataset, y_dataset, test_size=0.3, shuffle=True)
  83. return X_train, y_train, X_test, y_test, problem_size
  84. def main():
  85. parser = argparse.ArgumentParser(description="Train and find best filters to use for model")
  86. parser.add_argument('--data', type=str, help='open ml dataset filename prefix', required=True)
  87. parser.add_argument('--every_ls', type=int, help='train every ls surrogate model', default=50) # default value
  88. parser.add_argument('--k_division', type=int, help='number of expected sub surrogate model', default=20)
  89. parser.add_argument('--k_dynamic', type=int, help='specify if indices for each sub surrogate model are changed or not for each training', default=0, choices=[0, 1])
  90. parser.add_argument('--k_random', type=int, help='specify if split is random or not', default=1, choices=[0, 1])
  91. parser.add_argument('--ils', type=int, help='number of total iteration for ils algorithm', required=True)
  92. parser.add_argument('--ls', type=int, help='number of iteration for Local Search algorithm', required=True)
  93. parser.add_argument('--generate_only', type=int, help='number of iteration for Local Search algorithm', default=0, choices=[0, 1])
  94. parser.add_argument('--output', type=str, help='output surrogate model name')
  95. args = parser.parse_args()
  96. p_data_file = args.data
  97. p_every_ls = args.every_ls
  98. p_k_division = args.k_division
  99. p_k_dynamic = bool(args.k_dynamic)
  100. p_k_random = bool(args.k_random)
  101. p_ils_iteration = args.ils
  102. p_ls_iteration = args.ls
  103. p_generate_only = bool(args.generate_only)
  104. p_output = args.output
  105. # load data from file and get problem size
  106. X_train, y_train, X_test, y_test, problem_size = loadDataset(p_data_file)
  107. # create `logs` folder if necessary
  108. if not os.path.exists(cfg.output_logs_folder):
  109. os.makedirs(cfg.output_logs_folder)
  110. logging.basicConfig(format='%(asctime)s %(message)s', filename='data/logs/{0}.log'.format(p_output), level=logging.DEBUG)
  111. # init solution (`n` attributes)
  112. def init():
  113. return BinarySolution([], problem_size).random(validator)
  114. # define evaluate function here (need of data information)
  115. def evaluate(solution):
  116. start = datetime.datetime.now()
  117. # get indices of filters data to use (filters selection from solution)
  118. indices = []
  119. for index, value in enumerate(solution._data):
  120. if value == 1:
  121. indices.append(index)
  122. print(f'Training SVM with {len(indices)} from {len(solution._data)} available features')
  123. # keep only selected filters from solution
  124. x_train_filters = X_train[:, indices]
  125. x_test_filters = X_test[ :, indices]
  126. # model = mdl.get_trained_model(p_choice, x_train_filters, y_train_filters)
  127. model = train_model(x_train_filters, y_train)
  128. y_test_model = model.predict(x_test_filters)
  129. y_test_predict = [ 1 if x > 0.5 else 0 for x in y_test_model ]
  130. test_roc_auc = roc_auc_score(y_test, y_test_predict)
  131. end = datetime.datetime.now()
  132. diff = end - start
  133. print("Real evaluation took: {}, score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
  134. return test_roc_auc
  135. def sub_evaluate(solution, index_number, targeted_indices):
  136. start = datetime.datetime.now()
  137. # get indices of filters data to use (filters selection from solution)
  138. indices = []
  139. for index, value in enumerate(solution._data):
  140. if value == 1:
  141. indices.append(targeted_indices[index])
  142. print(f'Training sub-model SVM n°{index_number} with {len(indices)} from {len(solution._data)} available features')
  143. # keep only selected filters from solution
  144. x_train_filters = X_train[:, indices]
  145. x_test_filters = X_test[ :, indices]
  146. # model = mdl.get_trained_model(p_choice, x_train_filters, y_train_filters)
  147. model = train_model(x_train_filters, y_train)
  148. y_test_model = model.predict(x_test_filters)
  149. y_test_predict = [ 1 if x > 0.5 else 0 for x in y_test_model ]
  150. test_roc_auc = roc_auc_score(y_test, y_test_predict)
  151. end = datetime.datetime.now()
  152. diff = end - start
  153. print(f"Real sub-evaluation n°{index_number} took: {divmod(diff.days * 86400 + diff.seconds, 60)}, score found: {test_roc_auc}")
  154. return test_roc_auc
  155. # build all output folder and files based on `output` name
  156. backup_model_folder = os.path.join(cfg.output_backup_folder, p_output)
  157. surrogate_output_model = os.path.join(cfg.output_surrogates_model_folder, p_output)
  158. surrogate_output_data = os.path.join(cfg.output_surrogates_data_folder, p_output)
  159. if not os.path.exists(backup_model_folder):
  160. os.makedirs(backup_model_folder)
  161. if not os.path.exists(cfg.output_surrogates_model_folder):
  162. os.makedirs(cfg.output_surrogates_model_folder)
  163. if not os.path.exists(cfg.output_surrogates_data_folder):
  164. os.makedirs(cfg.output_surrogates_data_folder)
  165. backup_file_path = os.path.join(backup_model_folder, p_output + '.csv')
  166. ucb_backup_file_path = os.path.join(backup_model_folder, p_output + '_ucbPolicy.csv')
  167. surrogate_backup_file_path = os.path.join(cfg.output_surrogates_data_folder, p_output + '_train.csv')
  168. surrogate_k_indices_backup_file_path = os.path.join(cfg.output_surrogates_data_folder, p_output + '_k_indices.csv')
  169. # prepare optimization algorithm (only use of mutation as only ILS are used here, and local search need only local permutation)
  170. operators = [SimpleBinaryMutation(), SimpleMutation()]
  171. #policy = UCBPolicy(operators)
  172. policy = RandomPolicy(operators)
  173. # custom start surrogate variable based on problem size
  174. p_start = int(problem_size / p_k_division * 2) # 2 \times number of features for each sub-model
  175. # fixed minimal number of real evaluations
  176. if p_start < 50:
  177. p_start = 50
  178. print(f'Starting using surrogate after {p_start} reals training')
  179. # custom ILS for surrogate use
  180. algo = ILSMultiSpecificSurrogate(initalizer=init,
  181. evaluator=evaluate, # same evaluator by defadefaultult, as we will use the surrogate function
  182. sub_evaluator=sub_evaluate,
  183. operators=operators,
  184. policy=policy,
  185. validator=validator,
  186. output_log_surrogates=os.path.join(cfg.output_surrogates_data_folder, 'logs', p_output),
  187. surrogates_file_path=surrogate_output_model,
  188. start_train_surrogates=p_start, # start learning and using surrogate after 1000 real evaluation
  189. solutions_folder=surrogate_output_data,
  190. ls_train_surrogates=p_every_ls, # retrain surrogate every `x` iteration
  191. k_division=p_k_division,
  192. k_dynamic=p_k_dynamic,
  193. k_random=p_k_random,
  194. generate_only=p_generate_only,
  195. maximise=True)
  196. algo.addCallback(BasicCheckpoint(every=1, filepath=backup_file_path))
  197. #algo.addCallback(UCBCheckpoint(every=1, filepath=ucb_backup_file_path))
  198. algo.addCallback(SurrogateCheckpoint(every=p_ls_iteration, filepath=surrogate_backup_file_path)) # try every LS like this
  199. algo.addCallback(MultiSurrogateCheckpoint(every=p_ls_iteration, filepath=surrogate_k_indices_backup_file_path)) # try every LS like this
  200. bestSol = algo.run(p_ils_iteration, p_ls_iteration)
  201. # print best solution found
  202. print("Found ", bestSol)
  203. # save model information into .csv file
  204. if not os.path.exists(cfg.results_information_folder):
  205. os.makedirs(cfg.results_information_folder)
  206. filename_path = os.path.join(cfg.results_information_folder, cfg.optimization_attributes_result_filename)
  207. line_info = p_data_file + ';' + str(p_ils_iteration) + ';' + str(p_ls_iteration) + ';' + str(bestSol._data) + ';' + str(list(bestSol._data).count(1)) + ';' + str(bestSol.fitness())
  208. with open(filename_path, 'a') as f:
  209. f.write(line_info + '\n')
  210. print('Result saved into %s' % filename_path)
  211. if __name__ == "__main__":
  212. main()