add first version of LSTM 2D model

generate/generate_dataset_sequence_file.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jun 19 11:47:42 2019
+
+@author: jbuisine
+"""
+
+# main imports
+import sys, os, argparse
+import numpy as np
+import random
+
+# images processing imports
+from PIL import Image
+from ipfml.processing.segmentation import divide_in_blocks
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config  as cfg
+from modules.utils import data as dt
+from modules.classes.Transformation import Transformation
+
+# getting configuration information
+zone_folder             = cfg.zone_folder
+learned_folder          = cfg.learned_zones_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes values
+scenes_list             = cfg.scenes_names
+scenes_indices          = cfg.scenes_indices
+dataset_path            = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+features_choices        = cfg.features_choices_labels
+output_data_folder      = cfg.output_datasets
+
+generic_output_file_svd = '_random.csv'
+
+def generate_data_model(_filename, _transformations, _dataset_folder, _selected_zones, _sequence):
+
+    output_train_filename = os.path.join(output_data_folder, _filename, _filename + ".train")
+    output_test_filename = os.path.join(output_data_folder, _filename, _filename + ".val")
+
+    # create path if not exists
+    if not os.path.exists(os.path.join(output_data_folder, _filename)):
+        os.makedirs(os.path.join(output_data_folder, _filename))
+
+    train_file_data = []
+    test_file_data  = []
+
+    # specific number of zones (zones indices)
+    zones = np.arange(16)
+
+    # go ahead each scenes
+    for folder_scene in _selected_zones:
+
+        scene_path = os.path.join(_dataset_folder, folder_scene)
+
+        train_zones = _selected_zones[folder_scene]
+
+        for id_zone, index_folder in enumerate(zones):
+
+            index_str = str(index_folder)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+            
+            current_zone_folder = "zone" + index_str
+            zone_path = os.path.join(scene_path, current_zone_folder)
+
+            # custom path for interval of reconstruction and metric
+
+            features_path = []
+
+            for transformation in _transformations:
+                
+                # check if it's a static content and create augmented images if necessary
+                if transformation.getName() == 'static':
+                    
+                    # {sceneName}/zoneXX/static
+                    static_metric_path = os.path.join(zone_path, transformation.getName())
+
+                    # img.png
+                    image_name = transformation.getParam().split('/')[-1]
+
+                    # {sceneName}/zoneXX/static/img
+                    image_prefix_name = image_name.replace('.png', '')
+                    image_folder_path = os.path.join(static_metric_path, image_prefix_name)
+                    
+                    if not os.path.exists(image_folder_path):
+                        os.makedirs(image_folder_path)
+
+                    features_path.append(image_folder_path)
+
+                    # get image path to manage
+                    # {sceneName}/static/img.png
+                    transform_image_path = os.path.join(scene_path, transformation.getName(), image_name) 
+                    static_transform_image = Image.open(transform_image_path)
+
+                    static_transform_image_block = divide_in_blocks(static_transform_image, cfg.sub_image_size)[id_zone]
+
+                    dt.augmented_data_image(static_transform_image_block, image_folder_path, image_prefix_name)
+
+                else:
+                    metric_interval_path = os.path.join(zone_path, transformation.getTransformationPath())
+                    features_path.append(metric_interval_path)
+
+            # as labels are same for each metric
+            for label in os.listdir(features_path[0]):
+
+                label_features_path = []
+
+                for path in features_path:
+                    label_path = os.path.join(path, label)
+                    label_features_path.append(label_path)
+
+                # getting images list for each metric
+                features_images_list = []
+                    
+                for index_metric, label_path in enumerate(label_features_path):
+
+                    if _transformations[index_metric].getName() == 'static':
+                        # by default append nothing..
+                        features_images_list.append([])
+                    else:
+                        images = sorted(os.listdir(label_path))
+                        features_images_list.append(images)
+
+                sequence_data = []
+
+                # construct each line using all images path of each
+                for index_image in range(0, len(features_images_list[0])):
+                    
+                    images_path = []
+
+                    # get information about rotation and flip from first transformation (need to be a not static transformation)
+                    current_post_fix =  features_images_list[0][index_image].split(cfg.post_image_name_separator)[-1]
+
+                    # getting images with same index and hence name for each metric (transformation)
+                    for index_metric in range(0, len(features_path)):
+
+                        # custom behavior for static transformation (need to check specific image)
+                        if _transformations[index_metric].getName() == 'static':
+                            # add static path with selecting correct data augmented image
+                            image_name = _transformations[index_metric].getParam().split('/')[-1].replace('.png', '')
+                            img_path = os.path.join(features_path[index_metric], image_name + cfg.post_image_name_separator + current_post_fix)
+                            images_path.append(img_path)
+                        else:
+                            img_path = features_images_list[index_metric][index_image]
+                            images_path.append(os.path.join(label_features_path[index_metric], img_path))
+
+                    if label == cfg.noisy_folder:
+                        line = '1;'
+                    else:
+                        line = '0;'
+
+                    # add new data information into sequence
+                    sequence_data.append(images_path)
+
+                    if len(sequence_data) >= _sequence:
+                        
+                        # prepare whole line for LSTM model kind
+                        # keeping last noisy label
+
+                        for id_seq, seq_images_path in enumerate(sequence_data):
+                            # compute line information with all images paths
+                            for id_path, img_path in enumerate(seq_images_path):
+                                if id_path < len(seq_images_path) - 1:
+                                    line = line + img_path + '::'
+                                else:
+                                    line = line + img_path
+
+                            if id_seq < len(sequence_data) - 1:
+                                line += ';'
+                        
+                        line = line + '\n'
+
+                        if id_zone in train_zones:
+                            train_file_data.append(line)
+                        else:
+                            test_file_data.append(line)
+
+                        # remove first element (sliding window)
+                        del sequence_data[0]
+
+    train_file = open(output_train_filename, 'w')
+    test_file = open(output_test_filename, 'w')
+
+    random.shuffle(train_file_data)
+    random.shuffle(test_file_data)
+
+    for line in train_file_data:
+        train_file.write(line)
+
+    for line in test_file_data:
+        test_file.write(line)
+
+    train_file.close()
+    test_file.close()
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Compute specific dataset for model using of metric")
+
+    parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
+    parser.add_argument('--folder', type=str,
+                    help='folder where generated data are available',
+                    required=True) 
+    parser.add_argument('--features', type=str,
+                                     help="list of features choice in order to compute data",
+                                     default='svd_reconstruction, ipca_reconstruction',
+                                     required=True)
+    parser.add_argument('--params', type=str, 
+                                    help="list of specific param for each metric choice (See README.md for further information in 3D mode)", 
+                                    default='100, 200 :: 50, 25',
+                                    required=True)
+    parser.add_argument('--sequence', type=int, help='sequence length expected', required=True)
+    parser.add_argument('--size', type=str, 
+                                  help="Size of input images",
+                                  default="100, 100")
+    parser.add_argument('--selected_zones', type=str, help='file which contains all selected zones of scene', required=True)    
+
+    args = parser.parse_args()
+
+    p_filename   = args.output
+    p_folder     = args.folder
+    p_features   = list(map(str.strip, args.features.split(',')))
+    p_params     = list(map(str.strip, args.params.split('::')))
+    p_sequence   = args.sequence
+    p_size       = args.size # not necessary to split here
+    p_selected_zones = args.selected_zones
+
+    selected_zones = {}
+    with(open(p_selected_zones, 'r')) as f:
+
+        for line in f.readlines():
+
+            data = line.split(';')
+            del data[-1]
+            scene_name = data[0]
+            thresholds = data[1:]
+
+            selected_zones[scene_name] = [ int(t) for t in thresholds ]
+
+    # create list of Transformation
+    transformations = []
+
+    for id, feature in enumerate(p_features):
+
+        if feature not in features_choices:
+            raise ValueError("Unknown metric, please select a correct metric : ", features_choices)
+
+        transformations.append(Transformation(feature, p_params[id], p_size))
+
+    if transformations[0].getName() == 'static':
+        raise ValueError("The first transformation in list cannot be static")
+
+
+    # create database using img folder (generate first time only)
+    generate_data_model(p_filename, transformations, p_folder, selected_zones, p_sequence)
+
+if __name__== "__main__":
+    main()

generate/generate_reconstructed_folder.py

@@ -132,37 +132,38 @@ def generate_data(transformation, _dataset_path, _output, _human_thresholds, _re
                         label_path = os.path.join(label_path, cfg.noisy_folder)
 
                     # check if necessary to compute or not images
+                    # Disable use of data augmentation for the moment
                     # Data augmentation!
-                    rotations = [0, 90, 180, 270]
+                    # rotations = [0, 90, 180, 270]
 
                     #img_flip_labels = ['original', 'horizontal', 'vertical', 'both']
-                    img_flip_labels = ['original', 'horizontal']
+                    # img_flip_labels = ['original', 'horizontal']
 
-                    output_images_path = []
-                    check_path_exists = []
-                    # rotate and flip image to increase dataset size
-                    for id, flip_label in enumerate(img_flip_labels):
-                        for rotation in rotations:
-                            output_reconstructed_filename = img_path.split('/')[-1].replace('.png', '') + '_' + zones_folder[id_block] + cfg.post_image_name_separator
-                            output_reconstructed_filename = output_reconstructed_filename + flip_label + '_' + str(rotation) + '.png'
-                            output_reconstructed_path = os.path.join(label_path, output_reconstructed_filename)
+                    # output_images_path = []
+                    # check_path_exists = []
+                    # # rotate and flip image to increase dataset size
+                    # for id, flip_label in enumerate(img_flip_labels):
+                    #     for rotation in rotations:
+                    #         output_reconstructed_filename = img_path.split('/')[-1].replace('.png', '') + '_' + zones_folder[id_block] + cfg.post_image_name_separator
+                    #         output_reconstructed_filename = output_reconstructed_filename + flip_label + '_' + str(rotation) + '.png'
+                    #         output_reconstructed_path = os.path.join(label_path, output_reconstructed_filename)
 
-                            if os.path.exists(output_reconstructed_path):
-                                check_path_exists.append(True)
-                            else:
-                                check_path_exists.append(False)
+                    #         if os.path.exists(output_reconstructed_path):
+                    #             check_path_exists.append(True)
+                    #         else:
+                    #             check_path_exists.append(False)
 
-                            output_images_path.append(output_reconstructed_path)
+                    #         output_images_path.append(output_reconstructed_path)
 
                     # compute only if not exists or necessary to replace
-                    if _replace or not np.array(check_path_exists).all():
+                    # if _replace or not np.array(check_path_exists).all():
                         # compute image
                         # pass block to grey level
-                        output_block = transformation.getTransformedImage(block)
-                        output_block = np.array(output_block, 'uint8')
+                        # output_block = transformation.getTransformedImage(block)
+                        # output_block = np.array(output_block, 'uint8')
                         
-                        # current output image
-                        output_block_img = Image.fromarray(output_block)
+                        # # current output image
+                        # output_block_img = Image.fromarray(output_block)
 
                         #horizontal_img = output_block_img.transpose(Image.FLIP_LEFT_RIGHT)
                         #vertical_img = output_block_img.transpose(Image.FLIP_TOP_BOTTOM)
@@ -172,18 +173,44 @@ def generate_data(transformation, _dataset_path, _output, _human_thresholds, _re
                         #flip_images = [output_block_img, horizontal_img]
 
                         # Only current image img currenlty
-                        flip_images = [output_block_img]
+                        # flip_images = [output_block_img]
+
+                        # # rotate and flip image to increase dataset size
+                        # counter_index = 0 # get current path index
+                        # for id, flip in enumerate(flip_images):
+                        #     for rotation in rotations:
+
+                        #         if _replace or not check_path_exists[counter_index]:
+                        #             rotated_output_img = flip.rotate(rotation)
+                        #             rotated_output_img.save(output_images_path[counter_index])
+
+                        #         counter_index +=1
+                    
+                    if _replace:
+                        
+                        _, filename = os.path.split(img_path)
 
-                        # rotate and flip image to increase dataset size
-                        counter_index = 0 # get current path index
-                        for id, flip in enumerate(flip_images):
-                            for rotation in rotations:
+                        # build of output image filename
+                        filename = filename.replace('.png', '')
+                        filename_parts = filename.split('_')
 
-                                if _replace or not check_path_exists[counter_index]:
-                                    rotated_output_img = flip.rotate(rotation)
-                                    rotated_output_img.save(output_images_path[counter_index])
+                        # get samples : `00XXX`
+                        n_samples = filename_parts[2]
+                        del filename_parts[2]
+
+                        # `p3d_XXXXXX`
+                        output_reconstructed = '_'.join(filename_parts)
+
+                        output_reconstructed_filename = output_reconstructed + '_' + zones_folder[id_block] + '_' + n_samples + '.png'
+                        output_reconstructed_path = os.path.join(label_path, output_reconstructed_filename)
+
+                        output_block = transformation.getTransformedImage(block)
+                        output_block = np.array(output_block, 'uint8')
+                        
+                        # current output image
+                        output_block_img = Image.fromarray(output_block)
+                        output_block_img.save(output_reconstructed_path)
 
-                                counter_index +=1
 
                 write_progress((id_img + 1) / number_scene_image)
 

train_lstm_weighted.py

@@ -0,0 +1,266 @@
+# main imports
+import argparse
+import numpy as np
+import pandas as pd
+import os
+import ctypes
+from PIL import Image
+
+from keras import backend as K
+import matplotlib.pyplot as plt
+from ipfml import utils
+
+# dl imports
+from keras.layers import Dense, Dropout, LSTM, Embedding, GRU, BatchNormalization, ConvLSTM2D, Conv3D, Flatten
+from keras.preprocessing.sequence import pad_sequences
+from keras.models import Sequential
+from sklearn.metrics import roc_auc_score, accuracy_score
+import tensorflow as tf
+from keras import backend as K
+import sklearn
+from joblib import dump
+
+import custom_config as cfg
+
+
+def build_input(df, seq_norm):
+    """Convert dataframe to numpy array input with timesteps as float array
+    
+    Arguments:
+        df: {pd.Dataframe} -- Dataframe input
+        seq_norm: {bool} -- normalize or not seq input data by features
+    
+    Returns:
+        {np.ndarray} -- input LSTM data as numpy array
+    """
+
+    arr = []
+
+    # for each input line
+    for row in df.iterrows():
+
+        seq_arr = []
+
+        # for each sequence data input
+        for column in row[1]:
+
+            seq_elems = []
+
+            # for each element in sequence data
+            for img_path in column:
+                img = Image.open(img_path)
+                # seq_elems.append(np.array(img).flatten())
+                seq_elems.append(np.array(img))
+
+            #seq_arr.append(np.array(seq_elems).flatten())
+            seq_arr.append(np.array(seq_elems))
+            
+        arr.append(seq_arr)
+
+    arr = np.array(arr)
+    print(arr.shape)
+
+    # final_arr = []
+    # for v in arr:
+    #     v_data = []
+    #     for vv in v:
+    #         #scaled_vv = np.array(vv, 'float') - np.mean(np.array(vv, 'float'))
+    #         #v_data.append(scaled_vv)
+    #         v_data.append(vv)
+        
+    #     final_arr.append(v_data)
+    
+    final_arr = np.array(arr, 'float32')
+
+    # check if sequence normalization is used
+    if seq_norm:
+
+        if final_arr.ndim > 2:
+            n, s, f = final_arr.shape
+            for index, seq in enumerate(final_arr):
+                
+                for i in range(f):
+                    final_arr[index][:, i] = utils.normalize_arr_with_range(seq[:, i])
+
+            
+
+    return final_arr
+
+def create_model(_input_shape):
+    print ('Creating model...')
+    model = Sequential()
+    
+    # model.add(Conv3D(60, (1, 2, 2), input_shape=input_shape))
+    # model.add(Activation('relu'))
+    # model.add(MaxPooling3D(pool_size=(1, 2, 2)))
+
+    #model.add(Embedding(input_dim = 1000, output_dim = 50, input_length=input_length))
+    # model.add(ConvLSTM2D(filters=40, kernel_size=(3, 3), input_shape=input_shape, units=256, activation='sigmoid', recurrent_activation='hard_sigmoid'))
+    # model.add(Dropout(0.4))
+    # model.add(GRU(units=128, activation='sigmoid', recurrent_activation='hard_sigmoid'))
+    # model.add(Dropout(0.4))
+    # model.add(Dense(1, activation='sigmoid'))
+
+    model.add(ConvLSTM2D(filters=100, kernel_size=(3, 3),
+                   input_shape=_input_shape,
+                   padding='same', return_sequences=True))
+    model.add(BatchNormalization())
+    model.add(Dropout(0.4))
+
+    model.add(ConvLSTM2D(filters=50, kernel_size=(3, 3),
+                    padding='same', return_sequences=True))
+    model.add(BatchNormalization())
+    model.add(Dropout(0.4))
+
+    model.add(Conv3D(filters=20, kernel_size=(3, 3, 3),
+                activation='sigmoid',
+                padding='same', data_format='channels_last'))
+    model.add(Dropout(0.4))
+
+    model.add(Flatten())
+    model.add(Dense(512, activation='sigmoid'))
+    model.add(Dropout(0.4))
+    model.add(Dense(128, activation='sigmoid'))
+    model.add(Dropout(0.4))
+    model.add(Dense(1, activation='sigmoid'))
+    model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy'])
+
+    print ('Compiling...')
+    # model.compile(loss='binary_crossentropy',
+    #               optimizer='rmsprop',
+    #               metrics=['accuracy'])
+
+    return model
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Read and compute training of LSTM model")
+
+    parser.add_argument('--train', type=str, help='input train dataset')
+    parser.add_argument('--test', type=str, help='input test dataset')
+    parser.add_argument('--output', type=str, help='output model name')
+    parser.add_argument('--seq_norm', type=int, help='normalization sequence by features', choices=[0, 1])
+
+    args = parser.parse_args()
+
+    p_train        = args.train
+    p_test         = args.test
+    p_output       = args.output
+    p_seq_norm     = bool(args.seq_norm)
+
+
+    dataset_train = pd.read_csv(p_train, header=None, sep=';')
+    dataset_test = pd.read_csv(p_test, header=None, sep=';')
+
+    # getting weighted class over the whole dataset
+    noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 1]
+    not_noisy_df_train = dataset_train[dataset_train.iloc[:, 0] == 0]
+    nb_noisy_train = len(noisy_df_train.index)
+    nb_not_noisy_train = len(not_noisy_df_train.index)
+
+    noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 1]
+    not_noisy_df_test = dataset_test[dataset_test.iloc[:, 0] == 0]
+    nb_noisy_test = len(noisy_df_test.index)
+    nb_not_noisy_test = len(not_noisy_df_test.index)
+
+    noisy_samples = nb_noisy_test + nb_noisy_train
+    not_noisy_samples = nb_not_noisy_test + nb_not_noisy_train
+
+    total_samples = noisy_samples + not_noisy_samples
+
+    print('noisy', noisy_samples)
+    print('not_noisy', not_noisy_samples)
+    print('total', total_samples)
+
+    class_weight = {
+        0: noisy_samples / float(total_samples),
+        1: (not_noisy_samples / float(total_samples)),
+    }
+
+    # shuffle data
+    final_df_train = sklearn.utils.shuffle(dataset_train)
+    final_df_test = sklearn.utils.shuffle(dataset_test)
+
+    # split dataset into X_train, y_train, X_test, y_test
+    X_train = final_df_train.loc[:, 1:].apply(lambda x: x.astype(str).str.split('::'))
+    X_train = build_input(X_train, p_seq_norm)
+    y_train = final_df_train.loc[:, 0].astype('int')
+
+    X_test = final_df_test.loc[:, 1:].apply(lambda x: x.astype(str).str.split('::'))
+    X_test = build_input(X_test, p_seq_norm)
+    y_test = final_df_test.loc[:, 0].astype('int')
+
+    X_all = np.concatenate([X_train, X_test])
+    y_all = np.concatenate([y_train, y_test])
+
+    input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3], X_train.shape[4])
+    print('Training data input shape', input_shape)
+    model = create_model(input_shape)
+    model.summary()
+
+    print("Fitting model with custom class_weight", class_weight)
+    history = model.fit(X_train, y_train, batch_size=16, epochs=50, validation_split = 0.30, verbose=1, shuffle=True, class_weight=class_weight)
+
+    # list all data in history
+    print(history.history.keys())
+    # summarize history for accuracy
+    plt.plot(history.history['accuracy'])
+    plt.plot(history.history['val_accuracy'])
+    plt.title('model accuracy')
+    plt.ylabel('accuracy')
+    plt.xlabel('epoch')
+    plt.legend(['train', 'test'], loc='upper left')
+    plt.show()
+    # summarize history for loss
+    plt.plot(history.history['loss'])
+    plt.plot(history.history['val_loss'])
+    plt.title('model loss')
+    plt.ylabel('loss')
+    plt.xlabel('epoch')
+    plt.legend(['train', 'test'], loc='upper left')
+    plt.show()
+
+    train_score, train_acc = model.evaluate(X_train, y_train, batch_size=1)
+
+    print(train_acc)
+    y_train_predict = model.predict_classes(X_train)
+    y_test_predict = model.predict_classes(X_test)
+    y_all_predict = model.predict_classes(X_all)
+
+    print(y_train_predict)
+    print(y_test_predict)
+
+    auc_train = roc_auc_score(y_train, y_train_predict)
+    auc_test = roc_auc_score(y_test, y_test_predict)
+    auc_all = roc_auc_score(y_all, y_all_predict)
+
+    acc_train = accuracy_score(y_train, y_train_predict)
+    acc_test = accuracy_score(y_test, y_test_predict)
+    acc_all = accuracy_score(y_all, y_all_predict)
+    
+    print('Train ACC:', acc_train)
+    print('Train AUC', auc_train)
+    print('Test ACC:', acc_test)
+    print('Test AUC:', auc_test)
+    print('All ACC:', acc_all)
+    print('All AUC:', auc_all)
+
+
+    # save model results
+    if not os.path.exists(cfg.output_results_folder):
+        os.makedirs(cfg.output_results_folder)
+
+    results_filename = os.path.join(cfg.output_results_folder, cfg.results_filename)
+
+    with open(results_filename, 'a') as f:
+        f.write(p_output + ';' + str(acc_train) + ';' + str(auc_train) + ';' + str(acc_test) + ';' + str(auc_test) + '\n')
+
+    # save model using joblib
+    if not os.path.exists(cfg.output_models):
+        os.makedirs(cfg.output_models)
+
+    dump(model, os.path.join(cfg.output_models, p_output + '.joblib'))
+
+if __name__ == "__main__":
+    main()

train_model.py

@@ -76,7 +76,6 @@ def main():
 
     print("Number of chanels : ", n_chanels)
     img_width, img_height = [ int(s) for s in p_size ]
-    print(img_width, img_height)
 
     # specify the number of dimensions
     if K.image_data_format() == 'chanels_first':
@@ -175,8 +174,8 @@ def main():
         os.makedirs(model_backup_folder)
 
     # add of callback models
-    filepath = os.path.join(cfg.backup_model_folder, p_output, p_output + "-{auc:02f}-{val_auc:02f}__{epoch:02d}.hdf5")
-    checkpoint = ModelCheckpoint(filepath, monitor='val_auc', verbose=1, save_best_only=True, mode='max')
+    filepath = os.path.join(cfg.backup_model_folder, p_output, p_output + "-{accuracy:02f}-{val_accuracy:02f}__{epoch:02d}.hdf5")
+    checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
     callbacks_list = [checkpoint]