find_best_attributes_surrogate_openML_multi.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. # main imports
  2. import os
  3. import sys
  4. import argparse
  5. import pandas as pd
  6. import numpy as np
  7. import logging
  8. import datetime
  9. import random
  10. # model imports
  11. from sklearn.model_selection import train_test_split
  12. from sklearn.model_selection import GridSearchCV
  13. from sklearn.linear_model import LogisticRegression
  14. from sklearn.ensemble import RandomForestClassifier, VotingClassifier
  15. import joblib
  16. import sklearn
  17. import sklearn.svm as svm
  18. from sklearn.utils import shuffle
  19. from sklearn.metrics import roc_auc_score
  20. from sklearn.model_selection import cross_val_score
  21. from sklearn.preprocessing import MinMaxScaler
  22. # modules and config imports
  23. sys.path.insert(0, '') # trick to enable import of main folder module
  24. import custom_config as cfg
  25. import models as mdl
  26. from optimization.ILSMultiSurrogate import ILSMultiSurrogate
  27. from macop.solutions.BinarySolution import BinarySolution
  28. from macop.operators.mutators.SimpleMutation import SimpleMutation
  29. from macop.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
  30. from macop.operators.crossovers.SimpleCrossover import SimpleCrossover
  31. from macop.operators.crossovers.RandomSplitCrossover import RandomSplitCrossover
  32. from macop.operators.policies.UCBPolicy import UCBPolicy
  33. from macop.callbacks.BasicCheckpoint import BasicCheckpoint
  34. from macop.callbacks.UCBCheckpoint import UCBCheckpoint
  35. from optimization.callbacks.SurrogateCheckpoint import SurrogateCheckpoint
  36. from optimization.callbacks.MultiSurrogateCheckpoint import MultiSurrogateCheckpoint
  37. from sklearn.ensemble import RandomForestClassifier
  38. # avoid display of warning
  39. def warn(*args, **kwargs):
  40. pass
  41. import warnings
  42. warnings.filterwarnings("ignore", category=DeprecationWarning)
  43. warnings.warn = warn
  44. # default validator
  45. def validator(solution):
  46. # at least 5 attributes
  47. if list(solution._data).count(1) < 2:
  48. return False
  49. return True
  50. def train_model(X_train, y_train):
  51. #print ('Creating model...')
  52. # here use of SVM with grid search CV
  53. Cs = [0.001, 0.01, 0.1, 1, 10, 100]
  54. gammas = [0.001, 0.01, 0.1,10, 100]
  55. param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
  56. svc = svm.SVC(probability=True, class_weight='balanced')
  57. #clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
  58. clf = GridSearchCV(svc, param_grid, cv=4, verbose=0, n_jobs=-1)
  59. clf.fit(X_train, y_train)
  60. model = clf.best_estimator_
  61. return model
  62. def loadDataset(filename):
  63. ########################
  64. # 1. Get and prepare data
  65. ########################
  66. dataset = pd.read_csv(filename, sep=',')
  67. # change label as common
  68. min_label_value = min(dataset.iloc[:, -1])
  69. max_label_value = max(dataset.iloc[:, -1])
  70. dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(min_label_value, 0)
  71. dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(max_label_value, 1)
  72. X_dataset = dataset.iloc[:, :-1]
  73. y_dataset = dataset.iloc[:, -1]
  74. problem_size = len(X_dataset.columns)
  75. # min/max normalisation over feature
  76. # create a scaler object
  77. scaler = MinMaxScaler()
  78. # fit and transform the data
  79. X_dataset = np.array(pd.DataFrame(scaler.fit_transform(X_dataset), columns=X_dataset.columns))
  80. # prepare train, validation and test datasets
  81. X_train, X_test, y_train, y_test = train_test_split(X_dataset, y_dataset, test_size=0.3, shuffle=True)
  82. return X_train, y_train, X_test, y_test, problem_size
  83. def main():
  84. parser = argparse.ArgumentParser(description="Train and find best filters to use for model")
  85. parser.add_argument('--data', type=str, help='open ml dataset filename prefix', required=True)
  86. parser.add_argument('--every_ls', type=int, help='train every ls surrogate model', default=50) # default value
  87. parser.add_argument('--k_division', type=int, help='number of expected sub surrogate model', default=20)
  88. parser.add_argument('--k_dynamic', type=int, help='specify if indices for each sub surrogate model are changed or not for each training', default=0, choices=[0, 1])
  89. parser.add_argument('--k_random', type=int, help='specify if split is random or not', default=1, choices=[0, 1])
  90. parser.add_argument('--ils', type=int, help='number of total iteration for ils algorithm', required=True)
  91. parser.add_argument('--ls', type=int, help='number of iteration for Local Search algorithm', required=True)
  92. parser.add_argument('--generate_only', type=int, help='number of iteration for Local Search algorithm', default=0, choices=[0, 1])
  93. parser.add_argument('--output', type=str, help='output surrogate model name')
  94. args = parser.parse_args()
  95. p_data_file = args.data
  96. p_every_ls = args.every_ls
  97. p_k_division = args.k_division
  98. p_k_dynamic = bool(args.k_dynamic)
  99. p_k_random = bool(args.k_random)
  100. p_ils_iteration = args.ils
  101. p_ls_iteration = args.ls
  102. p_generate_only = bool(args.generate_only)
  103. p_output = args.output
  104. # load data from file and get problem size
  105. X_train, y_train, X_test, y_test, problem_size = loadDataset(p_data_file)
  106. # create `logs` folder if necessary
  107. if not os.path.exists(cfg.output_logs_folder):
  108. os.makedirs(cfg.output_logs_folder)
  109. logging.basicConfig(format='%(asctime)s %(message)s', filename='data/logs/{0}.log'.format(p_output), level=logging.DEBUG)
  110. # init solution (`n` attributes)
  111. def init():
  112. return BinarySolution([], problem_size).random(validator)
  113. # define evaluate function here (need of data information)
  114. def evaluate(solution):
  115. start = datetime.datetime.now()
  116. # get indices of filters data to use (filters selection from solution)
  117. indices = []
  118. for index, value in enumerate(solution._data):
  119. if value == 1:
  120. indices.append(index)
  121. print(f'Training SVM with {len(indices)} from {len(solution._data)} available features')
  122. # keep only selected filters from solution
  123. x_train_filters = X_train[:, indices]
  124. x_test_filters = X_test[ :, indices]
  125. # model = mdl.get_trained_model(p_choice, x_train_filters, y_train_filters)
  126. model = train_model(x_train_filters, y_train)
  127. y_test_model = model.predict(x_test_filters)
  128. y_test_predict = [ 1 if x > 0.5 else 0 for x in y_test_model ]
  129. test_roc_auc = roc_auc_score(y_test, y_test_predict)
  130. end = datetime.datetime.now()
  131. diff = end - start
  132. print("Real evaluation took: {}, score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
  133. return test_roc_auc
  134. # build all output folder and files based on `output` name
  135. backup_model_folder = os.path.join(cfg.output_backup_folder, p_output)
  136. surrogate_output_model = os.path.join(cfg.output_surrogates_model_folder, p_output)
  137. surrogate_output_data = os.path.join(cfg.output_surrogates_data_folder, p_output)
  138. if not os.path.exists(backup_model_folder):
  139. os.makedirs(backup_model_folder)
  140. if not os.path.exists(cfg.output_surrogates_model_folder):
  141. os.makedirs(cfg.output_surrogates_model_folder)
  142. if not os.path.exists(cfg.output_surrogates_data_folder):
  143. os.makedirs(cfg.output_surrogates_data_folder)
  144. backup_file_path = os.path.join(backup_model_folder, p_output + '.csv')
  145. ucb_backup_file_path = os.path.join(backup_model_folder, p_output + '_ucbPolicy.csv')
  146. surrogate_backup_file_path = os.path.join(cfg.output_surrogates_data_folder, p_output + '_train.csv')
  147. surrogate_k_indices_backup_file_path = os.path.join(cfg.output_surrogates_data_folder, p_output + '_k_indices.csv')
  148. # prepare optimization algorithm (only use of mutation as only ILS are used here, and local search need only local permutation)
  149. operators = [SimpleBinaryMutation(), SimpleMutation()]
  150. policy = UCBPolicy(operators)
  151. # define first line if necessary
  152. if not os.path.exists(surrogate_output_data):
  153. folder, _ = os.path.split(surrogate_output_data)
  154. if not os.path.exists(folder):
  155. os.makedirs(folder)
  156. with open(surrogate_output_data, 'w') as f:
  157. f.write('x;y\n')
  158. # custom start surrogate variable based on problem size
  159. p_start = int(0.5 * problem_size)
  160. # fixed minimal number of real evaluations
  161. if p_start < 50:
  162. p_start = 50
  163. print(f'Starting using surrogate after {p_start} reals training')
  164. # custom ILS for surrogate use
  165. algo = ILSMultiSurrogate(initalizer=init,
  166. evaluator=evaluate, # same evaluator by defadefaultult, as we will use the surrogate function
  167. operators=operators,
  168. policy=policy,
  169. validator=validator,
  170. output_log_surrogates=os.path.join(cfg.output_surrogates_data_folder, 'logs', p_output),
  171. surrogates_file_path=surrogate_output_model,
  172. start_train_surrogates=p_start, # start learning and using surrogate after 1000 real evaluation
  173. solutions_file=surrogate_output_data,
  174. ls_train_surrogates=p_every_ls, # retrain surrogate every `x` iteration
  175. k_division=p_k_division,
  176. k_dynamic=p_k_dynamic,
  177. k_random=p_k_random,
  178. generate_only=p_generate_only,
  179. maximise=True)
  180. algo.addCallback(BasicCheckpoint(every=1, filepath=backup_file_path))
  181. algo.addCallback(UCBCheckpoint(every=1, filepath=ucb_backup_file_path))
  182. algo.addCallback(SurrogateCheckpoint(every=p_ls_iteration, filepath=surrogate_backup_file_path)) # try every LS like this
  183. algo.addCallback(MultiSurrogateCheckpoint(every=p_ls_iteration, filepath=surrogate_k_indices_backup_file_path)) # try every LS like this
  184. bestSol = algo.run(p_ils_iteration, p_ls_iteration)
  185. # print best solution found
  186. print("Found ", bestSol)
  187. # save model information into .csv file
  188. if not os.path.exists(cfg.results_information_folder):
  189. os.makedirs(cfg.results_information_folder)
  190. filename_path = os.path.join(cfg.results_information_folder, cfg.optimization_attributes_result_filename)
  191. line_info = p_data_file + ';' + str(p_ils_iteration) + ';' + str(p_ls_iteration) + ';' + str(bestSol._data) + ';' + str(list(bestSol._data).count(1)) + ';' + str(bestSol.fitness)
  192. with open(filename_path, 'a') as f:
  193. f.write(line_info + '\n')
  194. print('Result saved into %s' % filename_path)
  195. if __name__ == "__main__":
  196. main()