Merge branch 'release/v0.3.7'

check_random_forest_perfomance.py

@@ -0,0 +1,144 @@
+# main imports
+import os
+import sys
+import argparse
+import pandas as pd
+import numpy as np
+import logging
+import datetime
+import random
+
+# model imports
+from sklearn.model_selection import train_test_split
+from sklearn.model_selection import GridSearchCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier, VotingClassifier
+from sklearn.feature_selection import SelectFromModel
+
+import joblib
+import sklearn.svm as svm
+from sklearn.utils import shuffle
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import cross_val_score
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+import models as mdl
+#from sklearn.ensemble import RandomForestClassifier
+
+def loadDataset(filename):
+
+    ########################
+    # 1. Get and prepare data
+    ########################
+    # scene_name; zone_id; image_index_end; label; data
+
+    dataset_train = pd.read_csv(filename + '.train', header=None, sep=";")
+    dataset_test = pd.read_csv(filename + '.test', header=None, sep=";")
+
+    # default first shuffle of data
+    dataset_train = shuffle(dataset_train)
+    dataset_test = shuffle(dataset_test)
+
+    # get dataset with equal number of classes occurences
+    noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 1]
+    not_noisy_df_train = dataset_train[dataset_train.iloc[:, 3] == 0]
+    #nb_noisy_train = len(noisy_df_train.index)
+
+    noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 1]
+    not_noisy_df_test = dataset_test[dataset_test.iloc[:, 3] == 0]
+    #nb_noisy_test = len(noisy_df_test.index)
+
+    # use of all data
+    final_df_train = pd.concat([not_noisy_df_train, noisy_df_train])
+    final_df_test = pd.concat([not_noisy_df_test, noisy_df_test])
+
+    # shuffle data another time
+    final_df_train = shuffle(final_df_train)
+    final_df_test = shuffle(final_df_test)
+
+    # use of the whole data set for training
+    x_dataset_train = final_df_train.iloc[:, 4:]
+    x_dataset_test = final_df_test.iloc[:, 4:]
+
+    y_dataset_train = final_df_train.iloc[:, 3]
+    y_dataset_test = final_df_test.iloc[:, 3]
+
+    return x_dataset_train, y_dataset_train, x_dataset_test, y_dataset_test
+
+
+def train_predict_random_forest(x_train, y_train, x_test, y_test):
+
+    print('Start training Random forest model')
+    start = datetime.datetime.now()
+    
+    # model = _get_best_model(x_train_filters, y_train_filters)
+    random_forest_model = RandomForestClassifier(n_estimators=500, class_weight='balanced', bootstrap=True, max_samples=0.75, n_jobs=-1)
+    random_forest_model = random_forest_model.fit(x_train, y_train)
+    
+    y_test_model = random_forest_model.predict(x_test)
+    test_roc_auc = roc_auc_score(y_test, y_test_model)
+
+    end = datetime.datetime.now()
+
+    diff = end - start
+
+    print("Evaluation took: {}, AUC score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
+
+    return random_forest_model
+
+
+def train_predict_selector(model, x_train, y_train, x_test, y_test):
+
+    start = datetime.datetime.now()
+
+    print("Using Select from model with Random Forest")
+    selector = SelectFromModel(estimator=model, prefit=True)
+    x_train_transformed = selector.transform(x_train)
+    x_test_transformed = selector.transform(x_test)
+
+    print('Previous shape:', x_train.shape)
+    print('New shape:', x_train_transformed.shape)
+
+    # using specific features
+    model = RandomForestClassifier(n_estimators=500, class_weight='balanced', bootstrap=True, max_samples=0.75, n_jobs=-1)
+    model = model.fit(x_train_transformed, y_train)
+
+    y_test_model= model.predict(x_test_transformed)
+    test_roc_auc = roc_auc_score(y_test, y_test_model)
+
+    end = datetime.datetime.now()
+
+    diff = end - start
+    print("Evaluation took: {}, AUC score found: {}".format(divmod(diff.days * 86400 + diff.seconds, 60), test_roc_auc))
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Train and find using all data to use for model")
+
+    parser.add_argument('--data', type=str, help='dataset filename prefix (without .train and .test)', required=True)
+    parser.add_argument('--output', type=str, help='output surrogate model name')
+
+    args = parser.parse_args()
+
+    p_data_file = args.data
+    p_output = args.output
+
+    print(p_data_file)
+
+    # load data from file
+    x_train, y_train, x_test, y_test = loadDataset(p_data_file)
+
+    # train classical random forest
+    random_forest_model = train_predict_random_forest(x_train, y_train, x_test, y_test)
+
+    # train using select from model
+    train_predict_selector(random_forest_model, x_train, y_train, x_test, y_test)
+
+
+
+
+if __name__ == "__main__":
+    main()

find_best_attributes_surrogate.py

@@ -41,7 +41,7 @@ from macop.policies.reinforcement import UCBPolicy
 from macop.callbacks.classicals import BasicCheckpoint
 from macop.callbacks.policies import UCBCheckpoint
 from optimization.callbacks.MultiPopCheckpoint import MultiPopCheckpoint
-
+from optimization.callbacks.SurrogateMonoCheckpoint import SurrogateMonoCheckpoint
 #from sklearn.ensemble import RandomForestClassifier
 
 # variables and parameters
@@ -204,6 +204,7 @@ def main():
 
     backup_file_path = os.path.join(backup_model_folder, p_output + '.csv')
     ucb_backup_file_path = os.path.join(backup_model_folder, p_output + '_ucbPolicy.csv')
+    surrogate_performanche_file_path = os.path.join(cfg.output_surrogates_data_folder, p_output + '_performance.csv')
 
     # prepare optimization algorithm (only use of mutation as only ILS are used here, and local search need only local permutation)
     operators = [SimpleBinaryMutation(), SimpleMutation(), RandomPopCrossover(), SimplePopCrossover()]
@@ -231,6 +232,7 @@ def main():
     
     algo.addCallback(MultiPopCheckpoint(every=1, filepath=backup_file_path))
     algo.addCallback(UCBCheckpoint(every=1, filepath=ucb_backup_file_path))
+    algo.addCallback(SurrogateMonoCheckpoint(every=1, filepath=surrogate_performanche_file_path))
 
     bestSol = algo.run(p_ils_iteration, p_ls_iteration)
 

optimization/ILSPopSurrogate.py

@@ -295,16 +295,19 @@ class ILSPopSurrogate(Algorithm):
                     self.add_to_surrogate(newSolution)
 
                     self.progress()
+                    
+                self.increaseEvaluation()
 
+                print(f'=================================================================')
                 print(f'Best solution found so far: {self.result.fitness}')
 
                 # check using specific dynamic criteria based on r^2
                 r_squared = self._surrogate.analysis.coefficient_of_determination(self._surrogate.surrogate)
                 mae = self._surrogate.analysis.mae(self._surrogate.surrogate)
                 training_surrogate_every = int(r_squared * self._ls_train_surrogate)
-                print(f"=> R^2 of surrogate is of {r_squared}. Retraining model every {training_surrogate_every} LS")
-                print(f"=> MAE of surrogate is of {mae}. Retraining model every {training_surrogate_every} LS")
-
+                print(f"=> R² of surrogate is of {r_squared}.")
+                print(f"=> MAE of surrogate is of {mae}.")
+                print(f'=> Retraining model every {training_surrogate_every} LS ({self._n_local_search % training_surrogate_every} of {training_surrogate_every})')
                 # avoid issue when lauching every each local search
                 if training_surrogate_every <= 0:
                     training_surrogate_every = 1

optimization/callbacks/SurrogateMonoCheckpoint.py

@@ -0,0 +1,91 @@
+"""Basic Checkpoint class implementation
+"""
+
+# main imports
+import os
+import logging
+import numpy as np
+
+# module imports
+from macop.callbacks.base import Callback
+from macop.utils.progress import macop_text, macop_line
+
+
+class SurrogateMonoCheckpoint(Callback):
+    """
+    SurrogateCheckpoint is used for logging training data information about surrogate
+
+    Attributes:
+        algo: {Algorithm} -- main algorithm instance reference
+        every: {int} -- checkpoint frequency used (based on number of evaluations)
+        filepath: {str} -- file path where checkpoints will be saved
+    """
+    def run(self):
+        """
+        Check if necessary to do backup based on `every` variable
+        """
+        # get current best solution
+        solution = self._algo._bestSolution
+        surrogate_analyser = self._algo._surrogate_analyser
+
+        # Do nothing is surrogate analyser does not exist
+        if surrogate_analyser is None:
+            return
+
+        currentEvaluation = self._algo.getGlobalEvaluation()
+
+        # backup if necessary
+        if currentEvaluation % self._every == 0:
+
+            logging.info(f"Surrogate analysis checkpoint is done into {self._filepath}")
+
+            solutionData = ""
+            solutionSize = len(solution._data)
+
+            for index, val in enumerate(solution._data):
+                solutionData += str(val)
+
+                if index < solutionSize - 1:
+                    solutionData += ' '
+
+            # get score of r² and mae
+
+            line = str(currentEvaluation) + ';' + str(surrogate_analyser._n_local_search) + ';' + str(surrogate_analyser._every_ls) + ';' + str(surrogate_analyser._time)  + ';' + str(surrogate_analyser._r2) \
+                + ';' + str(surrogate_analyser._mae) \
+                + ';' + solutionData + ';' + str(solution.fitness) + ';\n'
+
+            # check if file exists
+            if not os.path.exists(self._filepath):
+                with open(self._filepath, 'w') as f:
+                    f.write(line)
+            else:
+                with open(self._filepath, 'a') as f:
+                    f.write(line)
+
+    def load(self):
+        """
+        only load global n local search
+        """
+
+        if os.path.exists(self._filepath):
+
+            logging.info('Load n local search')
+            with open(self._filepath) as f:
+
+                # get last line and read data
+                lastline = f.readlines()[-1].replace(';\n', '')
+                data = lastline.split(';')
+
+                n_local_search = int(data[1])
+
+                # set k_indices into main algorithm
+                self._algo._total_n_local_search = n_local_search
+
+            print(macop_line(self._algo))
+            print(macop_text(self._algo, f'SurrogateMonoCheckpoint found from `{self._filepath}` file.'))
+
+        else:
+            print(macop_text(self._algo, 'No backup found...'))
+            logging.info("Can't load Surrogate backup... Backup filepath not valid in SurrogateCheckpoint")
+
+        print(macop_line(self._algo))

run_surrogate_rendering.sh

@@ -1,25 +1,25 @@
 #! /bin/bash
 
 # default param
-ILS=100000
+ILS=10000
 LS=100
 SS=50
-LENGTH=30
+LENGTH=32 # number of features
 POP=100
 ORDER=2
-TRAIN_EVERY=20
+TRAIN_EVERY=50
 
 
-output="rendering-attributes-ILS_${ILS}-POP_${POP}-LS_${LS}-SS_${SS}-SO_${ORDER}-SE_${TRAIN_EVERY}"
+#output="rendering-attributes-ILS_${ILS}-POP_${POP}-LS_${LS}-SS_${SS}-SO_${ORDER}-SE_${TRAIN_EVERY}"
 DATASET="rnn/data/datasets/features-selection-rendering-scaled/features-selection-rendering-scaled"
 
 for POP in {20,60,100};
 do
     for ORDER in {2,3};
     do
-        for LS in {100,500,1000};
+        for LS in {1000,5000,10000};
         do
-            output="rendering-attributes-ILS_${ILS}-POP_${POP}-LS_${LS}-SS_${SS}-SO_${ORDER}-SE_${TRAIN_EVERY}"
+            output="rendering-attributes-POP_${POP}-LS_${LS}-SS_${SS}-SO_${ORDER}-SE_${TRAIN_EVERY}"
             echo "Run optim attributes using: ${output}"
             python find_best_attributes_surrogate.py --data ${DATASET} --start_surrogate ${SS} --length 30 --ils ${ILS} --ls ${LS} --pop ${POP} --order ${ORDER} --train_every ${TRAIN_EVERY}  --output ${output}
         done