.gitignore

@@ -9,6 +9,7 @@ corr_indices
 metric_curves
 results
 .ipynb_checkpoints
+analysis/images
 
 # simulate_models.csv
 

check_random_forest_perfomance_rfe.py

@@ -0,0 +1,148 @@
+# main imports
+import os
+import sys
+import argparse
+import pandas as pd
+import numpy as np
+import logging
+import datetime
+import random
+
+# model imports
+from sklearn.model_selection import train_test_split
+from sklearn.model_selection import GridSearchCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier, VotingClassifier
+from sklearn.feature_selection import SelectFromModel
+
+import joblib
+import sklearn.svm as svm
+from sklearn.utils import shuffle
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import cross_val_score
+from sklearn.feature_selection import RFECV
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+import models as mdl
+#from sklearn.ensemble import RandomForestClassifier
+
+def loadDataset(filename):
+
+    ########################
+    # 1. Get and prepare data
+    ########################
+    # scene_name; zone_id; image_index_end; label; data
+
+    dataset_train = pd.read_csv(filename + '.train', header=None, sep=";")
+    dataset_test = pd.read_csv(filename + '.test', header=None, sep=";")
+
+    # default first shuffle of data
+    dataset_train = shuffle(dataset_train)
+    dataset_test = shuffle(dataset_test)
+
+    # get dataset with equal number of classes occurences
+    noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 1]
+    not_noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 0]
+    #nb_noisy_train = len(noisy_df_train.index)
+
+    noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 1]
+    not_noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 0]
+    #nb_noisy_test = len(noisy_df_test.index)
+
+    # use of all data
+    final_df_train = pd.concat([not_noisy_df_train, noisy_df_train])
+    final_df_test = pd.concat([not_noisy_df_test, noisy_df_test])
+
+    # shuffle data another time
+    final_df_train = shuffle(final_df_train)
+    final_df_test = shuffle(final_df_test)
+
+    # use of the whole data set for training
+    x_dataset_train = final_df_train.iloc[:, 4:]
+    x_dataset_test = final_df_test.iloc[:, 4:]
+
+    y_dataset_train = final_df_train.iloc[:, 3]
+    y_dataset_test = final_df_test.iloc[:, 3]
+
+    return x_dataset_train, y_dataset_train, x_dataset_test, y_dataset_test
+
+
+def train_predict_random_forest(x_train, y_train, x_test, y_test):
+
+    print('Start training Random forest model')
+    start = datetime.datetime.now()
+    
+    # model = _get_best_model(x_train_filters, y_train_filters)
+    random_forest_model = RandomForestClassifier(n_estimators=500, class_weight='balanced', bootstrap=True, max_samples=0.75, n_jobs=-1)
+
+    # No need to learn
+    random_forest_model = random_forest_model.fit(x_train, y_train)
+    
+    y_test_model = random_forest_model.predict(x_test)
+    test_roc_auc = roc_auc_score(y_test, y_test_model)
+
+    end = datetime.datetime.now()
+
+    diff = end - start
+
+    print("Evaluation took: {}, AUC score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
+
+    return random_forest_model
+
+
+def train_predict_selector(model, x_train, y_train, x_test, y_test):
+
+    start = datetime.datetime.now()
+
+    print("Using Select from model with Random Forest")
+    selector = RFECV(estimator=model, min_features_to_select=13, verbose=1, n_jobs=-1)
+    selector.fit(x_train, y_train)
+    x_train_transformed = selector.transform(x_train)
+    x_test_transformed = selector.transform(x_test)
+
+    print('Previous shape:', x_train.shape)
+    print('New shape:', x_train_transformed.shape)
+
+    # using specific features
+    model = RandomForestClassifier(n_estimators=500, class_weight='balanced', bootstrap=True, max_samples=0.75, n_jobs=-1)
+    model = model.fit(x_train_transformed, y_train)
+
+    y_test_model= model.predict(x_test_transformed)
+    test_roc_auc = roc_auc_score(y_test, y_test_model)
+
+    end = datetime.datetime.now()
+
+    diff = end - start
+    print("Evaluation took: {}, AUC score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Train and find using all data to use for model")
+
+    parser.add_argument('--data', type=str, help='dataset filename prefix (without .train and .test)', required=True)
+    parser.add_argument('--output', type=str, help='output surrogate model name')
+
+    args = parser.parse_args()
+
+    p_data_file = args.data
+    p_output = args.output
+
+    print(p_data_file)
+
+    # load data from file
+    x_train, y_train, x_test, y_test = loadDataset(p_data_file)
+
+    # train classical random forest
+    random_forest_model = train_predict_random_forest(x_train, y_train, x_test, y_test)
+
+    # train using select from model
+    train_predict_selector(random_forest_model, x_train, y_train, x_test, y_test)
+
+
+
+
+if __name__ == "__main__":
+    main()

find_best_attributes_no_surrogate.py

@@ -0,0 +1,285 @@
+# main imports
+import os
+import sys
+import argparse
+import pandas as pd
+import numpy as np
+import logging
+import datetime
+import random
+
+# model imports
+from sklearn.model_selection import train_test_split
+from sklearn.model_selection import GridSearchCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier, VotingClassifier
+
+import joblib
+import sklearn.svm as svm
+from sklearn.utils import shuffle
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import cross_val_score
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+import models as mdl
+
+from optimization.ILSPopNoSurrogate import ILSPopSurrogate
+from macop.solutions.discrete import BinarySolution
+from macop.evaluators.base import Evaluator
+
+from macop.operators.discrete.mutators import SimpleMutation
+from macop.operators.discrete.mutators import SimpleBinaryMutation
+from macop.operators.discrete.crossovers import SimpleCrossover
+from macop.operators.discrete.crossovers import RandomSplitCrossover
+from optimization.operators.SimplePopCrossover import SimplePopCrossover, RandomPopCrossover
+
+from macop.policies.reinforcement import UCBPolicy
+
+from macop.callbacks.classicals import BasicCheckpoint
+from macop.callbacks.policies import UCBCheckpoint
+from optimization.callbacks.MultiPopCheckpoint import MultiPopCheckpoint
+from optimization.callbacks.SurrogateMonoCheckpoint import SurrogateMonoCheckpoint
+#from sklearn.ensemble import RandomForestClassifier
+
+# variables and parameters
+models_list         = cfg.models_names_list
+
+from warnings import simplefilter
+simplefilter("ignore")
+
+# default validator
+def validator(solution):
+
+    # at least 5 attributes
+    if list(solution.data).count(1) < 5:
+        return False
+
+    return True
+
+def loadDataset(filename):
+
+    ########################
+    # 1. Get and prepare data
+    ########################
+    # scene_name; zone_id; image_index_end; label; data
+
+    dataset_train = pd.read_csv(filename + '.train', header=None, sep=";")
+    dataset_test = pd.read_csv(filename + '.test', header=None, sep=";")
+
+    # default first shuffle of data
+    dataset_train = shuffle(dataset_train)
+    dataset_test = shuffle(dataset_test)
+
+    # get dataset with equal number of classes occurences
+    noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 1]
+    not_noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 0]
+    #nb_noisy_train = len(noisy_df_train.index)
+
+    noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 1]
+    not_noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 0]
+    #nb_noisy_test = len(noisy_df_test.index)
+
+    # use of all data
+    final_df_train = pd.concat([not_noisy_df_train, noisy_df_train])
+    final_df_test = pd.concat([not_noisy_df_test, noisy_df_test])
+
+    # shuffle data another time
+    final_df_train = shuffle(final_df_train)
+    final_df_test = shuffle(final_df_test)
+
+    # use of the whole data set for training
+    x_dataset_train = final_df_train.iloc[:, 4:]
+    x_dataset_test = final_df_test.iloc[:, 4:]
+
+    y_dataset_train = final_df_train.iloc[:, 3]
+    y_dataset_test = final_df_test.iloc[:, 3]
+
+    return x_dataset_train, y_dataset_train, x_dataset_test, y_dataset_test
+
+def _get_best_model(X_train, y_train):
+
+    Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
+    gammas = [0.001, 0.01, 0.1, 5, 10, 100]
+    param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
+
+    svc = svm.SVC(probability=True, class_weight='balanced')
+    #clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
+    clf = GridSearchCV(svc, param_grid, cv=5, verbose=0, n_jobs=-1)
+
+    clf.fit(X_train, y_train)
+
+    model = clf.best_estimator_
+
+    return model
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Train and find best filters to use for model")
+
+    parser.add_argument('--data', type=str, help='dataset filename prefix (without .train and .test)', required=True)
+    parser.add_argument('--start_surrogate', type=int, help='number of evalution before starting surrogare model', required=True)
+    parser.add_argument('--train_every', type=int, help='max number of evalution before retraining surrogare model', required=True)
+    parser.add_argument('--length', type=int, help='max data length (need to be specify for evaluator)', required=True)
+    parser.add_argument('--pop', type=int, help='pop size', required=True)
+    parser.add_argument('--order', type=int, help='walsh order function', required=True)
+    parser.add_argument('--ils', type=int, help='number of total iteration for ils algorithm', required=True)
+    parser.add_argument('--ls', type=int, help='number of iteration for Local Search algorithm', required=True)
+    parser.add_argument('--output', type=str, help='output surrogate model name')
+
+    args = parser.parse_args()
+
+    p_data_file = args.data
+    p_length    = args.length
+    p_pop       = args.pop
+    p_order     = args.order
+    p_start     = args.start_surrogate
+    p_retrain   = args.train_every
+    p_ils_iteration = args.ils
+    p_ls_iteration  = args.ls
+    p_output = args.output
+
+    print(p_data_file)
+
+    # load data from file
+    x_train, y_train, x_test, y_test = loadDataset(p_data_file)
+
+    # create `logs` folder if necessary
+    if not os.path.exists(cfg.output_logs_folder):
+        os.makedirs(cfg.output_logs_folder)
+
+    logging.basicConfig(format='%(asctime)s %(message)s', filename='data/logs/{0}.log'.format(p_output), level=logging.DEBUG)
+
+    # init solution (`n` attributes)
+    def init():
+        return BinarySolution.random(p_length, validator)
+
+
+    class RandomForestEvaluator(Evaluator):
+
+        # define evaluate function here (need of data information)
+        def compute(self, solution):
+            start = datetime.datetime.now()
+
+            # get indices of filters data to use (filters selection from solution)
+            indices = []
+
+            for index, value in enumerate(solution.data): 
+                if value == 1: 
+                    indices.append(index) 
+
+            # keep only selected filters from solution
+            x_train_filters = self._data['x_train'].iloc[:, indices]
+            y_train_filters = self._data['y_train']
+            x_test_filters = self._data['x_test'].iloc[:, indices]
+            
+            # model = _get_best_model(x_train_filters, y_train_filters)
+            model = RandomForestClassifier(n_estimators=500, class_weight='balanced', bootstrap=True, max_samples=0.75, n_jobs=-1)
+            model = model.fit(x_train_filters, y_train_filters)
+            
+            y_test_model = model.predict(x_test_filters)
+            test_roc_auc = roc_auc_score(self._data['y_test'], y_test_model)
+
+            end = datetime.datetime.now()
+
+            diff = end - start
+
+            print("Real evaluation took: {}, score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
+
+            return test_roc_auc
+
+
+    # build all output folder and files based on `output` name
+    backup_model_folder = os.path.join(cfg.output_backup_folder, p_output)
+    surrogate_output_model = os.path.join(cfg.output_surrogates_model_folder, p_output)
+    surrogate_output_data = os.path.join(cfg.output_surrogates_data_folder, p_output)
+
+    if not os.path.exists(backup_model_folder):
+        os.makedirs(backup_model_folder)
+
+    if not os.path.exists(cfg.output_surrogates_model_folder):
+        os.makedirs(cfg.output_surrogates_model_folder)
+
+    if not os.path.exists(cfg.output_surrogates_data_folder):
+        os.makedirs(cfg.output_surrogates_data_folder)
+
+    backup_file_path = os.path.join(backup_model_folder, p_output + '.csv')
+    ucb_backup_file_path = os.path.join(backup_model_folder, p_output + '_ucbPolicy.csv')
+    surrogate_performanche_file_path = os.path.join(cfg.output_surrogates_data_folder, p_output + '_performance.csv')
+
+    # prepare optimization algorithm (only use of mutation as only ILS are used here, and local search need only local permutation)
+    operators = [SimpleBinaryMutation(), SimpleMutation(), RandomPopCrossover(), SimplePopCrossover()]
+    policy = UCBPolicy(operators, C=100, exp_rate=0.1)
+
+    # define first line if necessary
+    if not os.path.exists(surrogate_output_data):
+        with open(surrogate_output_data, 'w') as f:
+            f.write('x;y\n')
+
+    # custom ILS for surrogate use
+    algo = ILSPopSurrogate(initalizer=init, 
+                        evaluator=RandomForestEvaluator(data={'x_train': x_train, 'y_train': y_train, 'x_test': x_test, 'y_test': y_test}), # same evaluator by default, as we will use the surrogate function
+                        operators=operators, 
+                        policy=policy, 
+                        validator=validator,
+                        population_size=p_pop,
+                        surrogate_file_path=surrogate_output_model,
+                        start_train_surrogate=p_start, # start learning and using surrogate after 1000 real evaluation
+                        solutions_file=surrogate_output_data,
+                        walsh_order=p_order,
+                        inter_policy_ls_file=os.path.join(backup_model_folder, p_output + '_ls_ucbPolicy.csv'),
+                        ls_train_surrogate=p_retrain,
+                        maximise=True)
+    
+    algo.addCallback(MultiPopCheckpoint(every=1, filepath=backup_file_path))
+    algo.addCallback(UCBCheckpoint(every=1, filepath=ucb_backup_file_path))
+    algo.addCallback(SurrogateMonoCheckpoint(every=1, filepath=surrogate_performanche_file_path))
+
+    bestSol = algo.run(p_ils_iteration, p_ls_iteration)
+
+    # print best solution found
+    print("Found ", bestSol)
+
+    # save model information into .csv file
+    if not os.path.exists(cfg.results_information_folder):
+        os.makedirs(cfg.results_information_folder)
+
+    filename_path = os.path.join(cfg.results_information_folder, cfg.optimization_attributes_result_filename)
+
+    filters_counter = 0
+
+    # count number of filters
+    for index, item in enumerate(bestSol.data):
+        if index != 0 and index % 2 == 1:
+
+            # if two attributes are used
+            if item == 1 or bestSol.data[index - 1] == 1:
+                filters_counter += 1
+
+
+    line_info = p_output + ';' + p_data_file + ';' + str(bestSol.data) + ';' + str(list(bestSol.data).count(1)) + ';' + str(filters_counter) + ';' + str(bestSol.fitness)
+
+    # check if results are already saved...
+    already_saved = False
+
+    if os.path.exists(filename_path):
+        with open(filename_path, 'r') as f:
+            lines = f.readlines()
+
+            for line in lines:
+                output_name = line.split(';')[0]
+                
+                if p_output == output_name:
+                    already_saved = True
+
+    if not already_saved:
+        with open(filename_path, 'a') as f:
+            f.write(line_info + '\n')
+    
+    print('Result saved into %s' % filename_path)
+
+
+if __name__ == "__main__":
+    main()

models.py

@@ -25,8 +25,8 @@ from sklearn.model_selection import KFold, cross_val_score
 
 def _get_best_model(X_train, y_train):
 
-    Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
-    gammas = [0.001, 0.01, 0.1, 5, 10, 100]
+    Cs = [0.01, 0.1, 10, 100, 1000]
+    gammas = [0.01, 0.1, 10, 100]
     param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
 
     svc = svm.SVC(probability=True, class_weight='balanced')

optimization/ILSPopNoSurrogate.py

@@ -0,0 +1,374 @@
+"""Iterated Local Search Algorithm implementation using surrogate as fitness approximation
+"""
+
+# main imports
+import os
+import logging
+import joblib
+import time
+import pandas as pd
+from sklearn.utils import shuffle
+
+# module imports
+from macop.algorithms.base import Algorithm
+from macop.evaluators.base import Evaluator
+from macop.operators.base import KindOperator
+from macop.policies.reinforcement import UCBPolicy
+
+from macop.callbacks.policies import UCBCheckpoint
+
+from .LSNoSurrogate import LocalSearchSurrogate
+from .utils.SurrogateAnalysis import SurrogateAnalysisMono
+
+from sklearn.linear_model import (LinearRegression, Lasso, Lars, LassoLars,
+                                    LassoCV, ElasticNet)
+
+from wsao.sao.problems.nd3dproblem import ND3DProblem
+from wsao.sao.surrogates.walsh import WalshSurrogate
+from wsao.sao.algos.fitter import FitterAlgo
+from wsao.sao.utils.analysis import SamplerAnalysis, FitterAnalysis, OptimizerAnalysis
+
+
+class LSSurrogateEvaluator(Evaluator):
+
+    # use of surrogate in order to evaluate solution
+    def compute(self, solution):
+        return self._data['surrogate'].surrogate.predict([solution.data])[0]
+        
+
+class ILSPopSurrogate(Algorithm):
+    """Iterated Local Search used to avoid local optima and increave EvE (Exploration vs Exploitation) compromise using surrogate
+
+
+    Attributes:
+        initalizer: {function} -- basic function strategy to initialize solution
+        evaluator: {function} -- basic function in order to obtained fitness (mono or multiple objectives)
+        operators: {[Operator]} -- list of operator to use when launching algorithm
+        policy: {Policy} -- Policy class implementation strategy to select operators
+        validator: {function} -- basic function to check if solution is valid or not under some constraints
+        maximise: {bool} -- specify kind of optimization problem 
+        currentSolution: {Solution} -- current solution managed for current evaluation
+        bestSolution: {Solution} -- best solution found so far during running algorithm
+        ls_iteration: {int} -- number of evaluation for each local search algorithm
+        population_size: {int} -- size of the population to manage
+        surrogate_file: {str} -- Surrogate model file to load (model trained using https://gitlab.com/florianlprt/wsao)
+        start_train_surrogate: {int} -- number of evaluation expected before start training and use surrogate
+        surrogate: {Surrogate} -- Surrogate model instance loaded
+        ls_train_surrogate: {int} -- Specify if we need to retrain our surrogate model (every Local Search)
+        solutions_file: {str} -- Path where real evaluated solutions are saved in order to train surrogate again
+        callbacks: {[Callback]} -- list of Callback class implementation to do some instructions every number of evaluations and `load` when initializing algorithm
+    """
+    def __init__(self,
+                 initalizer,
+                 evaluator,
+                 operators,
+                 policy,
+                 validator,
+                 population_size,
+                 surrogate_file_path,
+                 start_train_surrogate,
+                 ls_train_surrogate,
+                 walsh_order,
+                 inter_policy_ls_file,
+                 solutions_file,
+                 maximise=True,
+                 parent=None):
+
+        # set real evaluator as default
+        super().__init__(initalizer, evaluator, operators, policy,
+                validator, maximise, parent)
+
+        self._n_local_search = 0
+        self._ls_local_search = 0
+        self._main_evaluator = evaluator
+
+        self._surrogate_file_path = surrogate_file_path
+        self._start_train_surrogate = start_train_surrogate
+
+        self._surrogate_evaluator = None
+        self._surrogate_analyser = None
+
+        self._ls_train_surrogate = ls_train_surrogate
+        self._solutions_file = solutions_file
+
+        self._walsh_order = walsh_order
+        self._inter_policy_ls_file = inter_policy_ls_file
+
+        # default population values
+        self.population_size = population_size
+        self.population = []
+
+        for _ in range(self.population_size):
+            self.population.append(None)
+
+    def train_surrogate(self):
+        """Retrain if necessary the whole surrogate fitness approximation function
+        """
+        # Following https://gitlab.com/florianlprt/wsao, we re-train the model
+        # ---------------------------------------------------------------------------
+        # cli_restart.py problem=nd3d,size=30,filename="data/statistics_extended_svdn" \
+        #        model=lasso,alpha=1e-5 \
+        #        surrogate=walsh,order=3 \
+        #        algo=fitter,algo_restarts=10,samplefile=stats_extended.csv \
+        #        sample=1000,step=10 \
+        #        analysis=fitter,logfile=out_fit.csv
+
+        problem = ND3DProblem(size=len(self._bestSolution.data)) # problem size based on best solution size (need to improve...)
+        model = Lasso(alpha=1e-5)
+        surrogate = WalshSurrogate(order=self._walsh_order, size=problem.size, model=model)
+        analysis = FitterAnalysis(logfile="train_surrogate.log", problem=problem)
+        algo = FitterAlgo(problem=problem, surrogate=surrogate, analysis=analysis, seed=problem.seed)
+
+        # data set
+        df = pd.read_csv(self._solutions_file, sep=';')
+        
+        # learning set and test set based on max last 1000 samples
+        max_samples = 1000
+
+        if df.x.count() < max_samples:
+            max_samples = df.x.count()
+
+        ntraining_samples = int(max_samples * 0.80)
+        
+        # extract reduced dataset if necessary
+        reduced_df = df.tail(max_samples)
+        reduced_df = shuffle(reduced_df)
+
+        # shuffle dataset
+        learn = reduced_df.tail(ntraining_samples)
+        test = reduced_df.drop(learn.index)
+
+        print("Start fitting again the surrogate model")
+        print(f'Using {ntraining_samples} samples of {max_samples} for train dataset')
+        for r in range(10):
+            print(f"Iteration n°{r}: for fitting surrogate")
+            algo.run_samples(learn=learn, test=test, step=10)
+
+        joblib.dump(algo, self._surrogate_file_path)
+
+
+    def load_surrogate(self):
+        """Load algorithm with surrogate model and create lambda evaluator function
+        """
+
+        # need to first train surrogate if not exist
+        if not os.path.exists(self._surrogate_file_path):
+            self.train_surrogate()
+
+        self._surrogate = joblib.load(self._surrogate_file_path)
+
+        # update evaluator function
+        self._surrogate_evaluator = LSSurrogateEvaluator(data={'surrogate': self._surrogate})
+
+    def add_to_surrogate(self, solution):
+
+        # save real evaluated solution into specific file for surrogate
+        with open(self._solutions_file, 'a') as f:
+
+            line = ""
+
+            for index, e in enumerate(solution._data):
+
+                line += str(e)
+                
+                if index < len(solution._data) - 1:
+                    line += ","
+
+            line += ";"
+            line += str(solution._score)
+
+            f.write(line + "\n")
+
+    def initRun(self):
+
+        fitness_scores = []
+        print('Initialisation of @population')
+        for i in range(len(self.population)):
+
+            print(f'  - solution [{(i+1)}] of {len(self.population)}')
+            if self.population[i] is None:
+                solution = self.initialiser()
+                solution.evaluate(self.evaluator)
+
+                self.population[i] = solution
+                self.add_to_surrogate(solution)
+
+            self.increaseEvaluation()
+
+            fitness_scores.append(self.population[i].fitness)
+
+        print('Best solution @initialisation')
+        self._bestSolution = self.population[fitness_scores.index(max(fitness_scores))]
+
+
+    def run(self, evaluations, ls_evaluations=100):
+        """
+        Run the iterated local search algorithm using local search (EvE compromise)
+
+        Args:
+            evaluations: {int} -- number of global evaluations for ILS
+            ls_evaluations: {int} -- number of Local search evaluations (default: 100)
+
+        Returns:
+            {Solution} -- best solution found
+        """
+
+        # by default use of mother method to initialize variables
+        super().run(evaluations)
+
+        # enable resuming for ILS
+        self.resume()
+
+        # initialize current solution
+        self.initRun()
+
+        # count number of surrogate obtained and restart using real evaluations done
+        nsamples = None
+        with open(self._solutions_file, 'r') as f:
+            nsamples = len(f.readlines()) - 1 # avoid header
+
+        if self.getGlobalEvaluation() < nsamples:
+            print(f'Restart using {nsamples} of {self._start_train_surrogate} real evaluations obtained')
+            self._numberOfEvaluations = nsamples
+
+        # if self._start_train_surrogate > self.getGlobalEvaluation():
+        
+        #     # get `self.start_train_surrogate` number of real evaluations and save it into surrogate dataset file
+        #     # using randomly generated solutions (in order to cover seearch space)
+        #     while self._start_train_surrogate > self.getGlobalEvaluation():
+                
+        #         newSolution = self.initialiser()
+
+        #         # evaluate new solution
+        #         newSolution.evaluate(self.evaluator)
+
+        #         # add it to surrogate pool
+        #         self.add_to_surrogate(newSolution)
+
+        #         self.increaseEvaluation()
+
+        # train surrogate on real evaluated solutions file
+        # self.train_surrogate()
+        # self.load_surrogate()
+
+        # local search algorithm implementation
+        while not self.stop():
+
+            # set current evaluator based on used or not of surrogate function
+            self.evaluator = self._surrogate_evaluator if self._start_train_surrogate <= self.getGlobalEvaluation() else self._main_evaluator
+
+            for i in range(len(self.population)):
+
+                # pass only Mutators operators for local search
+                selected_operators = [ op for op in self._operators if op._kind == KindOperator.MUTATOR ]
+
+                ls_policy = UCBPolicy(selected_operators, C=100, exp_rate=0.1)
+                # create new local search instance
+                # passing global evaluation param from ILS
+                ls = LocalSearchSurrogate(self.initialiser,
+                            self.evaluator,
+                            selected_operators,
+                            ls_policy,
+                            self.validator,
+                            self._maximise,
+                            parent=self,
+                            verbose=False)
+
+                ls.addCallback(UCBCheckpoint(every=1, filepath=self._inter_policy_ls_file))
+
+                # create current new solution using policy and custom algorithm init
+                ls._currentSolution = self.policy.apply(self.population[i])
+                ls.result = ls._currentSolution
+
+                # add same callbacks
+                #for callback in self._callbacks:
+                #    ls.addCallback(callback)
+
+                # create and search solution from local search
+                newSolution = ls.run(ls_evaluations)
+
+                # if better solution than currently, replace it (solution saved in training pool, only if surrogate process is in a second process step)
+                # Update : always add new solution into surrogate pool, not only if solution is better
+                #if self.isBetter(newSolution) and self.start_train_surrogate < self.getGlobalEvaluation():
+                # if self._start_train_surrogate <= self.getGlobalEvaluation():
+
+                # if better solution found from local search, retrained the found solution and test again
+                # without use of surrogate
+                # fitness_score = self._main_evaluator.compute(newSolution)
+                
+                # self.increaseEvaluation() # dot not add evaluation
+
+                # newSolution.fitness = fitness_score
+
+                # if solution is really better after real evaluation, then we replace
+                if self.isBetter(newSolution):
+                    self.result = newSolution
+
+                # update population
+                if self.population[i].fitness < newSolution.fitness:
+                    self.population[i] = newSolution
+
+                self.add_to_surrogate(newSolution)
+
+                self.progress()
+                    
+                self.increaseEvaluation()
+
+                print(f'=================================================================')
+                print(f'Best solution found so far: {self.result.fitness}')
+
+                # # check using specific dynamic criteria based on r^2
+                # r_squared = self._surrogate.analysis.coefficient_of_determination(self._surrogate.surrogate)
+                # mae = self._surrogate.analysis.mae(self._surrogate.surrogate)
+                # training_surrogate_every = int(r_squared * self._ls_train_surrogate)
+                # print(f"=> R² of surrogate is of {r_squared}.")
+                # print(f"=> MAE of surrogate is of {mae}.")
+                
+                # # avoid issue when lauching every each local search
+                # if training_surrogate_every <= 0:
+                #     training_surrogate_every = 1
+
+                # print(f'=> Retraining model every {training_surrogate_every} LS ({self._ls_local_search % training_surrogate_every} of {training_surrogate_every})')
+
+
+                # increase number of local search done
+                self._n_local_search += 1
+                # self._ls_local_search += 1
+
+                # # check if necessary or not to train again surrogate
+                # if self._ls_local_search % training_surrogate_every == 0 and self._start_train_surrogate <= self.getGlobalEvaluation():
+
+                #     # train again surrogate on real evaluated solutions file
+                #     start_training = time.time()
+                #     self.train_surrogate()
+                #     training_time = time.time() - start_training
+
+                #     self._surrogate_analyser = SurrogateAnalysisMono(training_time, training_surrogate_every, r_squared, mae, self.getGlobalMaxEvaluation(), self._n_local_search)
+
+                #     # reload new surrogate function
+                #     self.load_surrogate()
+
+                #     # reinit ls search
+                #     self._ls_local_search = 0
+
+                self.information()
+
+        logging.info(f"End of {type(self).__name__}, best solution found {self._bestSolution}")
+
+        self.end()
+        return self._bestSolution
+
+    def addCallback(self, callback):
+        """Add new callback to algorithm specifying usefull parameters
+
+        Args:
+            callback: {Callback} -- specific Callback instance
+        """
+        # specify current main algorithm reference
+        if self.getParent() is not None:
+            callback.setAlgo(self.getParent())
+        else:
+            callback.setAlgo(self)
+
+        # set as new
+        self._callbacks.append(callback)

optimization/LSNoSurrogate.py

@@ -0,0 +1,102 @@
+"""Local Search algorithm
+"""
+
+# main imports
+import logging
+
+# module imports
+from macop.algorithms.base import Algorithm
+
+
+class LocalSearchSurrogate(Algorithm):
+    """Local Search with surrogate used as exploitation optimization algorithm
+
+    Attributes:
+        initalizer: {function} -- basic function strategy to initialize solution
+        evaluator: {function} -- basic function in order to obtained fitness (mono or multiple objectives)
+        operators: {[Operator]} -- list of operator to use when launching algorithm
+        policy: {Policy} -- Policy class implementation strategy to select operators
+        validator: {function} -- basic function to check if solution is valid or not under some constraints
+        maximise: {bool} -- specify kind of optimization problem 
+        currentSolution: {Solution} -- current solution managed for current evaluation
+        bestSolution: {Solution} -- best solution found so far during running algorithm
+        callbacks: {[Callback]} -- list of Callback class implementation to do some instructions every number of evaluations and `load` when initializing algorithm
+    """
+    def run(self, evaluations):
+        """
+        Run the local search algorithm
+
+        Args:
+            evaluations: {int} -- number of Local search evaluations
+            
+        Returns:
+            {Solution} -- best solution found
+        """
+
+        # by default use of mother method to initialize variables
+        super().run(evaluations)
+
+        # do not use here the best solution known (default use of initRun and current solution)
+        # if self.parent:
+        #     self.bestSolution = self.parent.bestSolution
+
+        # initialize current solution
+        # self.initRun()
+
+        for callback in self._callbacks:
+            callback.load()
+
+        solutionSize = self._currentSolution.size
+
+        # local search algorithm implementation
+        while not self.stop():
+
+            for _ in range(solutionSize):
+
+                # update current solution using policy
+                newSolution = self.update(self._currentSolution)
+
+                # if better solution than currently, replace it
+                if self.isBetter(newSolution):
+                    self._bestSolution = newSolution
+
+                # increase number of evaluations
+                self.increaseEvaluation()
+
+                # self.progress()
+                for callback in self._callbacks:
+                    callback.run()
+
+                self._parent.add_to_surrogate(newSolution)
+
+                logging.info(f"---- Current {newSolution} - SCORE {newSolution.fitness}")
+
+                # add to surrogate pool file if necessary (using ILS parent reference)
+                # if self.parent.start_train_surrogate >= self.getGlobalEvaluation():
+                #     self.parent.add_to_surrogate(newSolution)
+
+                # stop algorithm if necessary
+                if self.stop():
+                    break
+
+            # after applying local search on currentSolution, we switch into new local area using known current bestSolution
+            self._currentSolution = self._bestSolution
+
+        logging.info(f"End of {type(self).__name__}, best solution found {self._bestSolution}")
+
+        return self._bestSolution
+
+    def addCallback(self, callback):
+        """Add new callback to algorithm specifying usefull parameters
+
+        Args:
+            callback: {Callback} -- specific Callback instance
+        """
+        # specify current main algorithm reference
+        if self._parent is not None:
+            callback.setAlgo(self._parent)
+        else:
+            callback.setAlgo(self)
+
+        # set as new
+        self._callbacks.append(callback)

run_no_surrogate_rendering.sh

@@ -0,0 +1,32 @@
+#! /bin/bash
+
+# default param
+ILS=10
+LS=100
+SS=1050 # Keep aware of surrogate never started
+LENGTH=32 # number of features
+POP=20
+ORDER=1
+TRAIN_EVERY=10
+
+
+#output="rendering-attributes-ILS_${ILS}-POP_${POP}-LS_${LS}-SS_${SS}-SO_${ORDER}-SE_${TRAIN_EVERY}"
+DATASET="rnn/data/datasets/features-selection-rendering-scaled/features-selection-rendering-scaled"
+
+for run in {1,2,3,4,5};
+do
+    # for POP in {20,60,100};
+    # do
+        #for ORDER in {1,2};
+        #for ORDER in {1};
+        #do
+            #for LS in {100,500,1000};
+            #for LS in {10};
+            #do
+                output="no-rendering-attributes-POP_${POP}-LS_${LS}-RUN_${run}"
+                echo "Run optim attributes using: ${output}"
+                python find_best_attributes_no_surrogate.py --data ${DATASET} --start_surrogate ${SS} --length 32 --ils ${ILS} --ls ${LS} --pop ${POP} --order ${ORDER} --train_every ${TRAIN_EVERY}  --output ${output}
+            #done
+        #done
+    # done
+done

run_surrogate_rendering.sh

@@ -1,11 +1,11 @@
 #! /bin/bash
 
 # default param
-ILS=2000
+ILS=1000
 LS=100
 SS=50
 LENGTH=32 # number of features
-POP=100
+POP=20
 ORDER=2
 TRAIN_EVERY=10
 
@@ -13,18 +13,18 @@ TRAIN_EVERY=10
 #output="rendering-attributes-ILS_${ILS}-POP_${POP}-LS_${LS}-SS_${SS}-SO_${ORDER}-SE_${TRAIN_EVERY}"
 DATASET="rnn/data/datasets/features-selection-rendering-scaled/features-selection-rendering-scaled"
 
-for run in {1,2,3,4,5,6,7,8,9,10};
+for run in {1,2,3,4,5};
 do
-    for POP in {20,60,100};
-    do
-        for ORDER in {1,2,3};
+    # for POP in {20,60,100};
+    # do
+        for ORDER in {1,2};
         do
-            for LS in {1000,5000,10000};
+            for LS in {100,500,1000};
             do
                 output="rendering-attributes-POP_${POP}-LS_${LS}-SS_${SS}-SO_${ORDER}-SE_${TRAIN_EVERY}-RUN_${run}"
                 echo "Run optim attributes using: ${output}"
                 python find_best_attributes_surrogate.py --data ${DATASET} --start_surrogate ${SS} --length 32 --ils ${ILS} --ls ${LS} --pop ${POP} --order ${ORDER} --train_every ${TRAIN_EVERY}  --output ${output}
             done
         done
-    done
+    # done
 done

train_model_attributes.py

@@ -12,7 +12,7 @@ from sklearn.ensemble import RandomForestClassifier, VotingClassifier
 import joblib
 import sklearn.svm as svm
 from sklearn.utils import shuffle
-from sklearn.metrics import accuracy_score, f1_score
+from sklearn.metrics import accuracy_score, roc_auc_score
 from sklearn.model_selection import cross_val_score
 
 # modules and config imports
@@ -28,45 +28,33 @@ models_list         = cfg.models_names_list
 current_dirpath     = os.getcwd()
 output_model_folder = os.path.join(current_dirpath, saved_models_folder)
 
-
-def main():
-
-    parser = argparse.ArgumentParser(description="Train SKLearn model and save it into .joblib file")
-
-    parser.add_argument('--data', type=str, help='dataset filename prefiloc (without .train and .test)')
-    parser.add_argument('--output', type=str, help='output file name desired for model (without .joblib extension)')
-    parser.add_argument('--choice', type=str, help='model choice from list of choices', choices=models_list)
-    parser.add_argument('--solution', type=str, help='Data of solution to specify filters to use')
-
-    args = parser.parse_args()
-
-    p_data_file = args.data
-    p_output    = args.output
-    p_choice    = args.choice
-    p_solution  = list(map(int, args.solution.split(' ')))
-
-    if not os.path.exists(output_model_folder):
-        os.makedirs(output_model_folder)
+def loadDataset(filename, n_step):
 
     ########################
     # 1. Get and prepare data
     ########################
-    dataset_train = pd.read_csv(p_data_file + '.train', header=None, sep=";")
-    dataset_test = pd.read_csv(p_data_file + '.test', header=None, sep=";")
+    # scene_name; zone_id; image_index_end; label; data
+
+    dataset_train = pd.read_csv(filename + '.train', header=None, sep=";")
+    dataset_test = pd.read_csv(filename + '.test', header=None, sep=";")
 
     # default first shuffle of data
     dataset_train = shuffle(dataset_train)
     dataset_test = shuffle(dataset_test)
 
+    dataset_train = dataset_train[dataset_train.iloc[:, 2] % n_step == 0]
+    dataset_test = dataset_test[dataset_test.iloc[:, 2] % n_step == 0]
+
     # get dataset with equal number of classes occurences
-    noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 1]
-    not_noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 0]
-    nb_noisy_train = len(noisy_df_train.index)
+    noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 1]
+    not_noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 0]
+    #nb_noisy_train = len(noisy_df_train.index)
 
-    noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 1]
-    not_noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 0]
-    nb_noisy_test = len(noisy_df_test.index)
+    noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 1]
+    not_noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 0]
+    #nb_noisy_test = len(noisy_df_test.index)
 
+    # use of all data
     final_df_train = pd.concat([not_noisy_df_train, noisy_df_train])
     final_df_test = pd.concat([not_noisy_df_test, noisy_df_test])
 
@@ -74,15 +62,40 @@ def main():
     final_df_train = shuffle(final_df_train)
     final_df_test = shuffle(final_df_test)
 
-    final_df_train_size = len(final_df_train.index)
-    final_df_test_size = len(final_df_test.index)
-
     # use of the whole data set for training
-    x_dataset_train = final_df_train.iloc[:,1:]
-    x_dataset_test = final_df_test.iloc[:,1:]
+    x_dataset_train = final_df_train.iloc[:, 4:]
+    x_dataset_test = final_df_test.iloc[:, 4:]
+
+    y_dataset_train = final_df_train.iloc[:, 3]
+    y_dataset_test = final_df_test.iloc[:, 3]
 
-    y_dataset_train = final_df_train.iloc[:,0]
-    y_dataset_test = final_df_test.iloc[:,0]
+    return x_dataset_train, y_dataset_train, x_dataset_test, y_dataset_test
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Train SKLearn model and save it into .joblib file")
+
+    parser.add_argument('--data', type=str, help='dataset filename prefiloc (without .train and .test)', required=True)
+    parser.add_argument('--output', type=str, help='output file name desired for model (without .joblib extension)', required=True)
+    parser.add_argument('--choice', type=str, help='model choice from list of choices', choices=models_list, required=True)
+    parser.add_argument('--step', type=int, help='step number of samples expected', default=20)
+    parser.add_argument('--solution', type=str, help='Data of solution to specify filters to use')
+
+    args = parser.parse_args()
+
+    p_data_file = args.data
+    p_output    = args.output
+    p_step      = args.step
+    p_choice    = args.choice
+    p_solution  = list(map(int, args.solution.split(' ')))
+
+    if not os.path.exists(output_model_folder):
+        os.makedirs(output_model_folder)
+
+    ########################
+    # 1. Get and prepare data
+    ########################
+    x_dataset_train, y_dataset_train, x_dataset_test, y_dataset_test = loadDataset(p_data_file, p_step)
 
     # get indices of filters data to use (filters selection from solution)
     indices = []
@@ -92,17 +105,21 @@ def main():
         if value == 1: 
             indices.append(index) 
 
-    print(indices)
+    print(f'Selected indices are: {indices}')
+    print(f"Train dataset size {len(x_dataset_train)}")
+    print(f"Test dataset size {len(x_dataset_test)}")
 
     x_dataset_train = x_dataset_train.iloc[:, indices]
     x_dataset_test =  x_dataset_test.iloc[:, indices]
 
+    print()
+
+    return
     #######################
     # 2. Construction of the model : Ensemble model structure
     #######################
 
     print("-------------------------------------------")
-    print("Train dataset size: ", final_df_train_size)
     model = mdl.get_trained_model(p_choice, x_dataset_train, y_dataset_train)
 
     #######################
@@ -112,40 +129,28 @@ def main():
     print("Accuracy: %0.2f (+/- %0.2f)" % (val_scores.mean(), val_scores.std() * 2))
 
     ######################
-    # 4. Test : Validation and test dataset from .test dataset
+    # 4. Metrics
     ######################
 
-    # we need to specify validation size to 20% of whole dataset
-    val_set_size = int(final_df_train_size/3)
-    test_set_size = val_set_size
-
-    total_validation_size = val_set_size + test_set_size
-
-    if final_df_test_size > total_validation_size:
-        x_dataset_test = x_dataset_test[0:total_validation_size]
-        y_dataset_test = y_dataset_test[0:total_validation_size]
-
-    X_test, X_val, y_test, y_val = train_test_split(x_dataset_test, y_dataset_test, test_size=0.5, random_state=1)
-
-    y_test_model = model.predict(X_test)
-    y_val_model = model.predict(X_val)
+    y_train_model = model.predict(x_dataset_train)
+    y_test_model = model.predict(x_dataset_test)
 
-    val_accuracy = accuracy_score(y_val, y_val_model)
-    test_accuracy = accuracy_score(y_test, y_test_model)
+    train_accuracy = accuracy_score(y_dataset_train, y_train_model)
+    test_accuracy = accuracy_score(y_dataset_test, y_test_model)
 
-    val_f1 = f1_score(y_val, y_val_model)
-    test_f1 = f1_score(y_test, y_test_model)
+    train_auc = roc_auc_score(y_dataset_train, y_train_model)
+    test_auc = roc_auc_score(y_dataset_test, y_test_model)
 
     ###################
     # 5. Output : Print and write all information in csv
     ###################
 
-    print("Validation dataset size ", val_set_size)
-    print("Validation: ", val_accuracy)
-    print("Validation F1: ", val_f1)
-    print("Test dataset size ", test_set_size)
-    print("Test: ", val_accuracy)
-    print("Test F1: ", test_f1)
+    print("Train dataset size ", len(x_dataset_train))
+    print("Train acc: ", train_accuracy)
+    print("Train AUC: ", train_auc)
+    print("Test dataset size ", len(x_dataset_test))
+    print("Test acc: ", test_accuracy)
+    print("Test AUC: ", test_auc)
 
     ##################
     # 6. Save model : create path if not exists