Merge branch 'release/v0.3.0' into master

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "modules"]
 	path = modules
-	url = https://github.com/prise-3d/Thesis-CommonModules.git
-[submodule "optimization"]
-	path = optimization
-	url = https://github.com/prise-3d/Thesis-OptimizationModules.git
+	url = https://github.com/prise-3d/Thesis-CommonModules.git

data_attributes.py

@@ -99,7 +99,9 @@ def get_image_features(data_type, block):
         bytes_data = np.array(block).tobytes()
         compress_data = gzip.compress(bytes_data)
 
-        data = np.append(data, sys.getsizeof(compress_data))
+        mo_size = sys.getsizeof(compress_data) / 1024.
+        go_size = mo_size / 1024.
+        data = np.append(data, go_size)
 
         lab_img = transform.get_LAB_L(block)
         arr = np.array(lab_img)

find_best_attributes.py

@@ -25,22 +25,24 @@ sys.path.insert(0, '') # trick to enable import of main folder module
 import custom_config as cfg
 import models as mdl
 
-from optimization.algorithms.IteratedLocalSearch import IteratedLocalSearch as ILS
-from optimization.solutions.BinarySolution import BinarySolution
+from macop.algorithms.mono.IteratedLocalSearch import IteratedLocalSearch as ILS
+from macop.solutions.BinarySolution import BinarySolution
 
-from optimization.operators.mutators.SimpleMutation import SimpleMutation
-from optimization.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
-from optimization.operators.crossovers.SimpleCrossover import SimpleCrossover
+from macop.operators.mutators.SimpleMutation import SimpleMutation
+from macop.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
+from macop.operators.crossovers.SimpleCrossover import SimpleCrossover
+from macop.operators.crossovers.RandomSplitCrossover import RandomSplitCrossover
 
-from optimization.operators.policies.RandomPolicy import RandomPolicy
+from macop.operators.policies.UCBPolicy import UCBPolicy
 
-from optimization.checkpoints.BasicCheckpoint import BasicCheckpoint
+from macop.callbacks.BasicCheckpoint import BasicCheckpoint
+from macop.callbacks.UCBCheckpoint import UCBCheckpoint
 
 # variables and parameters
 models_list         = cfg.models_names_list
 number_of_values    = 26
-ils_iteration       = 10
-ls_iteration        = 5
+ils_iteration       = 4000
+ls_iteration        = 10
 
 # default validator
 def validator(solution):
@@ -116,6 +118,11 @@ def main():
 
     logging.basicConfig(format='%(asctime)s %(message)s', filename='data/logs/%s.log' % p_data_file.split('/')[-1], level=logging.DEBUG)
 
+    # init solution (`n` attributes)
+    def init():
+        return BinarySolution([], 26
+        ).random(validator)
+
     # define evaluate function here (need of data information)
     def evaluate(solution):
 
@@ -146,21 +153,20 @@ def main():
 
         return test_roc_auc
 
-    # init solution (`n` attributes)
-    def init():
-        return BinarySolution([], number_of_values).random(validator)
-
     if not os.path.exists(cfg.output_backup_folder):
         os.makedirs(cfg.output_backup_folder)
 
     backup_file_path = os.path.join(cfg.output_backup_folder, p_data_file.split('/')[-1] + '.csv')
+    ucb_backup_file_path = os.path.join(cfg.output_backup_folder, p_data_file.split('/')[-1] + '_ucbPolicy.csv')
 
     # prepare optimization algorithm
-    updators = [SimpleBinaryMutation(), SimpleMutation(), SimpleCrossover()]
-    policy = RandomPolicy(updators)
+    operators = [SimpleBinaryMutation(), SimpleMutation(), SimpleCrossover(), RandomSplitCrossover()]
+    policy = UCBPolicy(operators)
 
-    algo = ILS(init, evaluate, updators, policy, validator, True)
-    algo.addCheckpoint(_class=BasicCheckpoint, _every=1, _filepath=backup_file_path)
+    algo = ILS(init, evaluate, operators, policy, validator, True)
+    
+    algo.addCallback(BasicCheckpoint(_every=1, _filepath=backup_file_path))
+    algo.addCallback(UCBCheckpoint(_every=1, _filepath=ucb_backup_file_path))
 
     bestSol = algo.run(ils_iteration, ls_iteration)
 

find_best_attributes_30.py

@@ -0,0 +1,200 @@
+# main imports
+import os
+import sys
+import argparse
+import pandas as pd
+import numpy as np
+import logging
+import datetime
+
+# model imports
+from sklearn.model_selection import train_test_split
+from sklearn.model_selection import GridSearchCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier, VotingClassifier
+
+import joblib
+import sklearn.svm as svm
+from sklearn.utils import shuffle
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import cross_val_score
+
+# modules and config imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+import models as mdl
+
+from macop.algorithms.mono.IteratedLocalSearch import IteratedLocalSearch as ILS
+from macop.solutions.BinarySolution import BinarySolution
+
+from macop.operators.mutators.SimpleMutation import SimpleMutation
+from macop.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
+from macop.operators.crossovers.SimpleCrossover import SimpleCrossover
+from macop.operators.crossovers.RandomSplitCrossover import RandomSplitCrossover
+
+from macop.operators.policies.UCBPolicy import UCBPolicy
+
+from macop.callbacks.BasicCheckpoint import BasicCheckpoint
+from macop.callbacks.UCBCheckpoint import UCBCheckpoint
+
+# variables and parameters
+models_list         = cfg.models_names_list
+number_of_values    = 30
+ils_iteration       = 4000
+ls_iteration        = 10
+
+# default validator
+def validator(solution):
+
+    if list(solution.data).count(1) < 5:
+        return False
+
+    return True
+
+def loadDataset(filename):
+
+    ########################
+    # 1. Get and prepare data
+    ########################
+    dataset_train = pd.read_csv(filename + '.train', header=None, sep=";")
+    dataset_test = pd.read_csv(filename + '.test', header=None, sep=";")
+
+    # default first shuffle of data
+    dataset_train = shuffle(dataset_train)
+    dataset_test = shuffle(dataset_test)
+
+    # get dataset with equal number of classes occurences
+    noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 1]
+    not_noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 0]
+    #nb_noisy_train = len(noisy_df_train.index)
+
+    noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 1]
+    not_noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 0]
+    #nb_noisy_test = len(noisy_df_test.index)
+
+    # use of all data
+    final_df_train = pd.concat([not_noisy_df_train, noisy_df_train])
+    final_df_test = pd.concat([not_noisy_df_test, noisy_df_test])
+
+    # shuffle data another time
+    final_df_train = shuffle(final_df_train)
+    final_df_test = shuffle(final_df_test)
+
+    # use of the whole data set for training
+    x_dataset_train = final_df_train.iloc[:,1:]
+    x_dataset_test = final_df_test.iloc[:,1:]
+
+    y_dataset_train = final_df_train.iloc[:,0]
+    y_dataset_test = final_df_test.iloc[:,0]
+
+    return x_dataset_train, y_dataset_train, x_dataset_test, y_dataset_test
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Train and find best filters to use for model")
+
+    parser.add_argument('--data', type=str, help='dataset filename prefix (without .train and .test)', required=True)
+    parser.add_argument('--choice', type=str, help='model choice from list of choices', choices=models_list, required=True)
+    parser.add_argument('--length', type=str, help='max data length (need to be specify for evaluator)', required=True)
+
+    args = parser.parse_args()
+
+    p_data_file = args.data
+    p_choice    = args.choice
+    p_length    = args.length
+
+    global number_of_values
+    number_of_values = p_length
+
+    print(p_data_file)
+
+    # load data from file
+    x_train, y_train, x_test, y_test = loadDataset(p_data_file)
+
+    # create `logs` folder if necessary
+    if not os.path.exists(cfg.output_logs_folder):
+        os.makedirs(cfg.output_logs_folder)
+
+    logging.basicConfig(format='%(asctime)s %(message)s', filename='data/logs/%s.log' % p_data_file.split('/')[-1], level=logging.DEBUG)
+
+    # init solution (`n` attributes)
+    def init():
+        return BinarySolution([], 30
+        ).random(validator)
+
+    # define evaluate function here (need of data information)
+    def evaluate(solution):
+
+        start = datetime.datetime.now()
+        # get indices of filters data to use (filters selection from solution)
+        indices = []
+
+        for index, value in enumerate(solution.data): 
+            if value == 1: 
+                indices.append(index) 
+
+        # keep only selected filters from solution
+        x_train_filters = x_train.iloc[:, indices]
+        y_train_filters = y_train
+        x_test_filters = x_test.iloc[:, indices]
+
+        # TODO : use of GPU implementation of SVM
+        model = mdl.get_trained_model(p_choice, x_train_filters, y_train_filters)
+        
+        y_test_model = model.predict(x_test_filters)
+        test_roc_auc = roc_auc_score(y_test, y_test_model)
+
+        end = datetime.datetime.now()
+
+        diff = end - start
+
+        print("Evaluation took :", divmod(diff.days * 86400 + diff.seconds, 60))
+
+        return test_roc_auc
+
+    if not os.path.exists(cfg.output_backup_folder):
+        os.makedirs(cfg.output_backup_folder)
+
+    backup_file_path = os.path.join(cfg.output_backup_folder, p_data_file.split('/')[-1] + '.csv')
+    ucb_backup_file_path = os.path.join(cfg.output_backup_folder, p_data_file.split('/')[-1] + '_ucbPolicy.csv')
+
+    # prepare optimization algorithm
+    operators = [SimpleBinaryMutation(), SimpleMutation(), SimpleCrossover(), RandomSplitCrossover()]
+    policy = UCBPolicy(operators)
+
+    algo = ILS(init, evaluate, operators, policy, validator, True)
+    
+    algo.addCallback(BasicCheckpoint(_every=1, _filepath=backup_file_path))
+    algo.addCallback(UCBCheckpoint(_every=1, _filepath=ucb_backup_file_path))
+
+    bestSol = algo.run(ils_iteration, ls_iteration)
+
+    # print best solution found
+    print("Found ", bestSol)
+
+    # save model information into .csv file
+    if not os.path.exists(cfg.results_information_folder):
+        os.makedirs(cfg.results_information_folder)
+
+    filename_path = os.path.join(cfg.results_information_folder, cfg.optimization_attributes_result_filename)
+
+    filters_counter = 0
+    # count number of filters
+    for index, item in enumerate(bestSol.data):
+        if index != 0 and index % 2 == 1:
+
+            # if two attributes are used
+            if item == 1 or bestSol.data[index - 1] == 1:
+                filters_counter += 1
+
+
+    line_info = p_data_file + ';' + str(ils_iteration) + ';' + str(ls_iteration) + ';' + str(bestSol.data) + ';' + str(list(bestSol.data).count(1)) + ';' + str(filters_counter) + ';' + str(bestSol.fitness())
+    with open(filename_path, 'a') as f:
+        f.write(line_info + '\n')
+    
+    print('Result saved into %s' % filename_path)
+
+
+if __name__ == "__main__":
+    main()

find_best_filters.py

@@ -24,16 +24,18 @@ sys.path.insert(0, '') # trick to enable import of main folder module
 import custom_config as cfg
 import models as mdl
 
-from optimization.algorithms.IteratedLocalSearch import IteratedLocalSearch as ILS
-from optimization.solutions.BinarySolution import BinarySolution
+from macop.macop.algorithms.mono.IteratedLocalSearch import IteratedLocalSearch as ILS
+from macop.macop.solutions.BinarySolution import BinarySolution
 
-from optimization.operators.mutators.SimpleMutation import SimpleMutation
-from optimization.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
-from optimization.operators.crossovers.SimpleCrossover import SimpleCrossover
+from macop.macop.operators.mutators.SimpleMutation import SimpleMutation
+from macop.macop.operators.mutators.SimpleBinaryMutation import SimpleBinaryMutation
+from macop.macop.operators.crossovers.SimpleCrossover import SimpleCrossover
+from macop.macop.operators.crossovers.RandomSplitCrossover import RandomSplitCrossover
 
-from optimization.operators.policies.RandomPolicy import RandomPolicy
+from macop.macop.operators.policies.UCBPolicy import UCBPolicy
 
-from optimization.checkpoints.BasicCheckpoint import BasicCheckpoint
+from macop.macop.callbacks.BasicCheckpoint import BasicCheckpoint
+from macop.macop.callbacks.UCBCheckpoint import UCBCheckpoint
 
 # variables and parameters
 models_list         = cfg.models_names_list
@@ -106,8 +108,8 @@ def main():
     x_train, y_train, x_test, y_test = loadDataset(p_data_file)
 
     # create `logs` folder if necessary
-    if not os.path.exists(cfg.logs_folder):
-        os.makedirs(cfg.logs_folder)
+    if not os.path.exists(cfg.output_logs_folder):
+        os.makedirs(cfg.output_logs_folder)
 
     logging.basicConfig(format='%(asctime)s %(message)s', filename='logs/%s.log' % p_data_file.split('/')[-1], level=logging.DEBUG)
 
@@ -134,17 +136,19 @@ def main():
 
         return test_roc_auc
 
-    if not os.path.exists(cfg.backup_folder):
-        os.makedirs(cfg.backup_folder)
+    if not os.path.exists(cfg.output_backup_folder):
+        os.makedirs(cfg.output_backup_folder)
 
-    backup_file_path = os.path.join(cfg.backup_folder, p_data_file.split('/')[-1] + '.csv')
+    backup_file_path = os.path.join(cfg.output_backup_folder, p_data_file.split('/')[-1] + '.csv')
+    ucb_backup_file_path = os.path.join(cfg.output_backup_folder, p_data_file.split('/')[-1] + '_ucbPolicy.csv')
 
     # prepare optimization algorithm
-    updators = [SimpleBinaryMutation(), SimpleMutation(), SimpleCrossover()]
-    policy = RandomPolicy(updators)
+    operators = [SimpleBinaryMutation(), SimpleMutation(), SimpleCrossover(), RandomSplitCrossover()]
+    policy = UCBPolicy(operators)
 
-    algo = ILS(init, evaluate, updators, policy, validator, True)
-    algo.addCheckpoint(_class=BasicCheckpoint, _every=1, _filepath=backup_file_path)
+    algo = ILS(init, evaluate, operators, policy, validator, True)
+    algo.addCallback(BasicCheckpoint(_every=1, _filepath=backup_file_path))
+    algo.addCallback(UCBCheckpoint(_every=1, _filepath=ucb_backup_file_path))
 
     bestSol = algo.run(ils_iteration, ls_iteration)
 

models.py

@@ -1,4 +1,6 @@
 # models imports
+import numpy as np
+
 from sklearn.model_selection import GridSearchCV
 from sklearn.linear_model import LogisticRegression
 from sklearn.ensemble import RandomForestClassifier, VotingClassifier
@@ -8,16 +10,18 @@ from sklearn.feature_selection import RFECV
 import sklearn.svm as svm
 from sklearn.metrics import accuracy_score
 from thundersvm import SVC
+from sklearn.model_selection import KFold, cross_val_score
+            
 
 # variables and parameters
 n_predict = 0
 
-def my_accuracy_scorer(*args):
-        global n_predict
-        score = accuracy_score(*args)
-        print('{0} - Score is {1}'.format(n_predict, score))
-        n_predict += 1
-        return score
+# def my_accuracy_scorer(*args):
+#         global n_predict
+#         score = accuracy_score(*args)
+#         print('{0} - Score is {1}'.format(n_predict, score))
+#         n_predict += 1
+#         return score
 
 def _get_best_model(X_train, y_train):
 
@@ -26,7 +30,8 @@ def _get_best_model(X_train, y_train):
     param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
 
     svc = svm.SVC(probability=True, class_weight='balanced')
-    clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
+    #clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
+    clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, n_jobs=-1)
 
     clf.fit(X_train, y_train)
 
@@ -41,12 +46,13 @@ def svm_model(X_train, y_train):
 
 def _get_best_gpu_model(X_train, y_train):
 
-    Cs = [0.001, 0.01, 0.1, 1, 2, 5, 10, 100, 1000]
-    gammas = [0.001, 0.01, 0.1, 1, 2, 5, 10, 100]
+    Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
+    gammas = [0.001, 0.01, 0.1, 5, 10, 100]
     param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}
 
     svc = SVC(probability=True, class_weight='balanced')
-    clf = GridSearchCV(svc, param_grid, cv=10, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
+    #clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, scoring=my_accuracy_scorer, n_jobs=-1)
+    clf = GridSearchCV(svc, param_grid, cv=5, verbose=1, n_jobs=-1)
 
     clf.fit(X_train, y_train)
 

optimization

@@ -1 +0,0 @@
-Subproject commit db3538337fc6fcce7796378d50ade263b1c103fd

requirements.txt

@@ -10,4 +10,5 @@ matplotlib
 path.py
 pandas
 opencv-python
-joblib
+joblib
+macop