find_best_attributes_surrogate_openML.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. # main imports
  2. import os
  3. import sys
  4. import argparse
  5. import pandas as pd
  6. import numpy as np
  7. import logging
  8. import datetime
  9. import random
  10. # model imports
  11. from sklearn.model_selection import train_test_split
  12. from sklearn.model_selection import GridSearchCV
  13. from sklearn.linear_model import LogisticRegression
  14. from sklearn.ensemble import RandomForestClassifier, VotingClassifier
  15. import joblib
  16. import sklearn
  17. import sklearn.svm as svm
  18. from sklearn.utils import shuffle
  19. from sklearn.metrics import roc_auc_score
  20. from sklearn.model_selection import cross_val_score
  21. from sklearn.preprocessing import MinMaxScaler
  22. # modules and config imports
  23. sys.path.insert(0, '') # trick to enable import of main folder module
  24. import custom_config as cfg
  25. import models as mdl
  26. from optimization.ILSSurrogate import ILSSurrogate
  27. from macop.solutions.BinarySolution import BinarySolution
  28. from macop.operators.mutators.SimpleMutation import SimpleMutation
  29. from macop.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
  30. from macop.operators.crossovers.SimpleCrossover import SimpleCrossover
  31. from macop.operators.crossovers.RandomSplitCrossover import RandomSplitCrossover
  32. from macop.operators.policies.UCBPolicy import UCBPolicy
  33. from macop.callbacks.BasicCheckpoint import BasicCheckpoint
  34. from macop.callbacks.UCBCheckpoint import UCBCheckpoint
  35. from optimization.callbacks.SurrogateCheckpoint import SurrogateCheckpoint
  36. from sklearn.ensemble import RandomForestClassifier
  37. # default validator
  38. def validator(solution):
  39. # at least 5 attributes
  40. if list(solution._data).count(1) < 2:
  41. return False
  42. return True
  43. def train_model(X_train, y_train):
  44. #print ('Creating model...')
  45. # here use of SVM with grid search CV
  46. Cs = [0.001, 0.01, 0.1, 1, 10, 100]
  47. gammas = [0.001, 0.01, 0.1,10, 100]
  48. param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
  49. svc = svm.SVC(probability=True, class_weight='balanced')
  50. #clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
  51. clf = GridSearchCV(svc, param_grid, cv=4, verbose=0, n_jobs=-1)
  52. clf.fit(X_train, y_train)
  53. model = clf.best_estimator_
  54. return model
  55. def loadDataset(filename):
  56. ########################
  57. # 1. Get and prepare data
  58. ########################
  59. dataset = pd.read_csv(filename, sep=',')
  60. # change label as common
  61. min_label_value = min(dataset.iloc[:, -1])
  62. max_label_value = max(dataset.iloc[:, -1])
  63. dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(min_label_value, 0)
  64. dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(max_label_value, 1)
  65. X_dataset = dataset.iloc[:, :-1]
  66. y_dataset = dataset.iloc[:, -1]
  67. problem_size = len(X_dataset.columns)
  68. # min/max normalisation over feature
  69. # create a scaler object
  70. scaler = MinMaxScaler()
  71. # fit and transform the data
  72. X_dataset = np.array(pd.DataFrame(scaler.fit_transform(X_dataset), columns=X_dataset.columns))
  73. # prepare train, validation and test datasets
  74. X_train, X_test, y_train, y_test = train_test_split(X_dataset, y_dataset, test_size=0.3, shuffle=True)
  75. return X_train, y_train, X_test, y_test, problem_size
  76. def main():
  77. parser = argparse.ArgumentParser(description="Train and find best filters to use for model")
  78. parser.add_argument('--data', type=str, help='open ml dataset filename prefix', required=True)
  79. parser.add_argument('--every_ls', type=int, help='train every ls surrogate model', default=50) # default value
  80. parser.add_argument('--ils', type=int, help='number of total iteration for ils algorithm', required=True)
  81. parser.add_argument('--ls', type=int, help='number of iteration for Local Search algorithm', required=True)
  82. parser.add_argument('--output', type=str, help='output surrogate model name')
  83. args = parser.parse_args()
  84. p_data_file = args.data
  85. p_every_ls = args.every_ls
  86. p_ils_iteration = args.ils
  87. p_ls_iteration = args.ls
  88. p_output = args.output
  89. # load data from file and get problem size
  90. X_train, y_train, X_test, y_test, problem_size = loadDataset(p_data_file)
  91. # create `logs` folder if necessary
  92. if not os.path.exists(cfg.output_logs_folder):
  93. os.makedirs(cfg.output_logs_folder)
  94. logging.basicConfig(format='%(asctime)s %(message)s', filename='data/logs/{0}.log'.format(p_output), level=logging.DEBUG)
  95. # init solution (`n` attributes)
  96. def init():
  97. return BinarySolution([], problem_size).random(validator)
  98. # define evaluate function here (need of data information)
  99. def evaluate(solution):
  100. start = datetime.datetime.now()
  101. # get indices of filters data to use (filters selection from solution)
  102. indices = []
  103. for index, value in enumerate(solution._data):
  104. if value == 1:
  105. indices.append(index)
  106. print(f'Training SVM with {len(indices)} from {len(solution._data)} available features')
  107. # keep only selected filters from solution
  108. x_train_filters = X_train[:, indices]
  109. x_test_filters = X_test[ :, indices]
  110. # model = mdl.get_trained_model(p_choice, x_train_filters, y_train_filters)
  111. model = train_model(x_train_filters, y_train)
  112. y_test_model = model.predict(x_test_filters)
  113. y_test_predict = [ 1 if x > 0.5 else 0 for x in y_test_model ]
  114. test_roc_auc = roc_auc_score(y_test, y_test_predict)
  115. end = datetime.datetime.now()
  116. diff = end - start
  117. print("Real evaluation took: {}, score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
  118. return test_roc_auc
  119. # build all output folder and files based on `output` name
  120. backup_model_folder = os.path.join(cfg.output_backup_folder, p_output)
  121. surrogate_output_model = os.path.join(cfg.output_surrogates_model_folder, p_output)
  122. surrogate_output_data = os.path.join(cfg.output_surrogates_data_folder, p_output)
  123. if not os.path.exists(backup_model_folder):
  124. os.makedirs(backup_model_folder)
  125. if not os.path.exists(cfg.output_surrogates_model_folder):
  126. os.makedirs(cfg.output_surrogates_model_folder)
  127. if not os.path.exists(cfg.output_surrogates_data_folder):
  128. os.makedirs(cfg.output_surrogates_data_folder)
  129. backup_file_path = os.path.join(backup_model_folder, p_output + '.csv')
  130. ucb_backup_file_path = os.path.join(backup_model_folder, p_output + '_ucbPolicy.csv')
  131. surrogate_backup_file_path = os.path.join(cfg.output_surrogates_data_folder, p_output + '_train.csv')
  132. # prepare optimization algorithm (only use of mutation as only ILS are used here, and local search need only local permutation)
  133. operators = [SimpleBinaryMutation(), SimpleMutation()]
  134. policy = UCBPolicy(operators)
  135. # define first line if necessary
  136. if not os.path.exists(surrogate_output_data):
  137. folder, _ = os.path.split(surrogate_output_data)
  138. if not os.path.exists(folder):
  139. os.makedirs(folder)
  140. with open(surrogate_output_data, 'w') as f:
  141. f.write('x;y\n')
  142. # custom start surrogate variable based on problem size
  143. p_start = int(0.5 * problem_size)
  144. # fixed limit
  145. if p_start < 50:
  146. p_start = 50
  147. print(f'Starting using surrogate after {p_start} reals training')
  148. # custom ILS for surrogate use
  149. algo = ILSSurrogate(initalizer=init,
  150. evaluator=evaluate, # same evaluator by defadefaultult, as we will use the surrogate function
  151. operators=operators,
  152. policy=policy,
  153. validator=validator,
  154. surrogate_file_path=surrogate_output_model,
  155. start_train_surrogate=p_start, # start learning and using surrogate after 1000 real evaluation
  156. solutions_file=surrogate_output_data,
  157. ls_train_surrogate=p_every_ls, # retrain surrogate every 5 iteration
  158. maximise=True)
  159. algo.addCallback(BasicCheckpoint(every=1, filepath=backup_file_path))
  160. algo.addCallback(UCBCheckpoint(every=1, filepath=ucb_backup_file_path))
  161. algo.addCallback(SurrogateCheckpoint(every=p_ls_iteration, filepath=surrogate_backup_file_path)) # try every LS like this
  162. bestSol = algo.run(p_ils_iteration, p_ls_iteration)
  163. # print best solution found
  164. print("Found ", bestSol)
  165. # save model information into .csv file
  166. if not os.path.exists(cfg.results_information_folder):
  167. os.makedirs(cfg.results_information_folder)
  168. filename_path = os.path.join(cfg.results_information_folder, cfg.optimization_attributes_result_filename)
  169. line_info = p_data_file + ';' + str(p_ils_iteration) + ';' + str(p_ls_iteration) + ';' + str(bestSol.data) + ';' + str(list(bestSol.data).count(1)) + ';' + str(bestSol.fitness())
  170. with open(filename_path, 'a') as f:
  171. f.write(line_info + '\n')
  172. print('Result saved into %s' % filename_path)
  173. if __name__ == "__main__":
  174. main()