Thesis-NoiseDetection-26-attributes/features_selection/run_method_openML.py

import os, argparse

import numpy as np
import pandas as pd

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score, accuracy_score
import sklearn.svm as svm

from methods import features_selection_list, features_selection_method


def train_model(X_train, y_train):

    print ('Creating model...')
    # here use of SVM with grid search CV
    Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
    gammas = [0.001, 0.01, 0.1,10, 100, 1000]
    param_grid = {'kernel':['rbf'], 'C': Cs, 'gamma' : gammas}

    svc = svm.SVC(probability=True, class_weight='balanced')
    clf = GridSearchCV(svc, param_grid, cv=2, verbose=1, n_jobs=-1)

    clf.fit(X_train, y_train)

    model = clf.best_estimator_

    return model

def loadDataset(filename):

    ########################
    # 1. Get and prepare data
    ########################
    dataset = pd.read_csv(filename, sep=',')

    # change label as common
    min_label_value = min(dataset.iloc[:, -1])
    max_label_value = max(dataset.iloc[:, -1])

    dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(min_label_value, 0)
    dataset.iloc[:, -1] = dataset.iloc[:, -1].replace(max_label_value, 1)

    X_dataset = dataset.iloc[:, :-1]
    y_dataset = dataset.iloc[:, -1]

    problem_size = len(X_dataset.columns)

    # min/max normalisation over feature
    # create a scaler object
    scaler = MinMaxScaler()
    # fit and transform the data
    X_dataset = np.array(pd.DataFrame(scaler.fit_transform(X_dataset), columns=X_dataset.columns))

    # prepare train, validation and test datasets
    X_train, X_test, y_train, y_test = train_test_split(X_dataset, y_dataset, test_size=0.3, shuffle=True)

    return X_train, y_train, X_test, y_test, problem_size


def main():

    parser = argparse.ArgumentParser(description="Get features extraction from specific method")

    parser.add_argument('--data', type=str, help='open ml dataset filename prefix', required=True)
    parser.add_argument('--method', type=str, help='method name to use', choices=features_selection_list, required=True)
    parser.add_argument('--params', type=str, help='params used for the current selected method', required=True)
    parser.add_argument('--ntrain', type=int, help='number of training in order to keep mean of score', default=1)
    parser.add_argument('--output', type=str, help='output features selection results')

    args = parser.parse_args()

    p_data_file = args.data
    p_method    = args.method
    p_params    = args.params
    p_ntrain    = args.ntrain
    p_output    = args.output

    # load data from file and get problem size
    X_train, y_train, X_test, y_test, problem_size = loadDataset(p_data_file)

    # extract indices selected features
    features_indices = features_selection_method(p_method, p_params, X_train, y_train, problem_size)

    print(f'Selected features {len(features_indices)} over {problem_size}')

    auc_scores = []
    acc_scores = []
    
    for i in range(p_ntrain):

        # new split of dataset
        X_train, y_train, X_test, y_test, problem_size = loadDataset(p_data_file)

        # get reduced dataset
        X_train_reduced = X_train[:, features_indices]
        X_test_reduced = X_test[:, features_indices]


        # get trained model over reduce dataset
        model = train_model(X_train_reduced, y_train)

        # get predicted labels over test dataset
        y_test_model = model.predict(X_test_reduced)
        y_test_predict = [ 1 if x > 0.5 else 0 for x in y_test_model ]
        test_roc_auc = roc_auc_score(y_test, y_test_predict)
        test_acc = accuracy_score(y_test, y_test_predict)

        print(f'Run n°{i}: {test_roc_auc} (AUC ROC)')

        # append score into list of run
        auc_scores.append(test_roc_auc)
        acc_scores.append(test_acc)

    mean_auc_score = sum(auc_scores) / len(auc_scores)
    mean_acc_score = sum(acc_scores) / len(acc_scores)

    var_acc_score = np.var(acc_scores)
    var_auc_score = np.var(auc_scores)

    std_acc_score = np.std(acc_scores)
    std_auc_score = np.std(auc_scores)

    print(f'Model performance using {p_method} (params: {p_params}) is of {mean_auc_score:.2f}')

    # now save trained model and params obtained
    header_line = 'dataset;method;params;ntrain;n_features;acc_test;auc_test;var_acc_test;var_auc_test;std_acc_test;std_auc_test;features_indices\n'
    data_line = f'{p_data_file};{p_method};{p_params};{p_ntrain};{len(features_indices)};{mean_acc_score};{mean_auc_score};{var_acc_score};{var_auc_score};{std_acc_score};{std_auc_score};{" ".join(list(map(str, features_indices)))}\n'

    output_folder, _ = os.path.split(p_output)

    if len(output_folder) > 0:
        if not os.path.exists(output_folder):
            os.makedirs(output_folder)

    if not os.path.exists(p_output):
        with open(p_output, 'w') as f:
            f.write(header_line)

    with open(p_output, 'a') as f:
        f.write(data_line)
    

if __name__ == "__main__":
    main()