Merge branch 'release/v0.0.9'

.gitignore

@@ -2,9 +2,14 @@
 data
 .python-version
 __pycache__
+.vscode
 
 # by default avoid model files and png files
 *.h5
 *.png
 !saved_models/*.h5
-!saved_models/*.png
+!saved_models/*.png
+
+# data
+learned_zones
+dataset

TODO.md

@@ -0,0 +1,25 @@
+# TODO :
+
+## Prépation des données
+
+- Séparer dans 2 dossiers les images (noisy, not noisy) 
+  - Par scène
+  - Par zone
+  - Par métrique [scene, zone]
+  
+- Transformer chaque image comme souhaitée (ici reconstruction SV avec 110 composantes faibles)
+- Pour chaque image ajouter sa forme sous 4 rotations (augmentation du nombre de données)
+
+## Chargement des données
+- Chargement de l'ensemble des images (association : "path", "label")
+- Mettre en place un équilibre de classes
+- Mélange de l'ensemble des données
+- Séparation des données (train, validation, test)
+
+## Conception du modèle
+- Mise en place d'un modèle CNN
+- Utilisation BatchNormalization / Dropout
+
+
+## Si non fonctionnel
+- Utilisation d'une approche transfer learning

classification_cnn_keras.py

@@ -1,244 +0,0 @@
-'''This script goes along the blog post
-"Building powerful image classification models using very little data"
-from blog.keras.io.
-```
-data/
-    train/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-    validation/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-```
-'''
-import sys, os, getopt
-import json
-
-from keras.preprocessing.image import ImageDataGenerator
-from keras.models import Sequential
-from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D
-from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
-from keras import backend as K
-from keras.utils import plot_model
-
-from ipfml import tf_model_helper
-
-# local functions import (metrics preprocessing)
-import preprocessing_functions
-
-##########################################
-# Global parameters (with default value) #
-##########################################
-img_width, img_height = 100, 100
-
-train_data_dir = 'data/train'
-validation_data_dir = 'data/validation'
-nb_train_samples = 7200
-nb_validation_samples = 3600
-epochs = 50
-batch_size = 16
-
-input_shape = (3, img_width, img_height)
-
-###########################################
-
-'''
-Method which returns model to train
-@return : DirectoryIterator
-'''
-def generate_model():
-
-    model = Sequential()
-
-    model.add(Conv2D(60, (2, 2), input_shape=input_shape))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-
-    model.add(Conv2D(40, (2, 2)))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-
-    model.add(Conv2D(20, (2, 2)))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-
-    model.add(Flatten())
-
-    model.add(Dense(140))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.3))
-
-    model.add(Dense(120))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.3))
-
-    model.add(Dense(80))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.2))
-
-    model.add(Dense(40))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.2))
-
-    model.add(Dense(20))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.2))
-
-    model.add(Dense(1))
-    model.add(Activation('sigmoid'))
-
-    model.compile(loss='binary_crossentropy',
-                  optimizer='rmsprop',
-                  metrics=['accuracy'])
-
-    return model
-
-'''
-Method which loads train data
-@return : DirectoryIterator
-'''
-def load_train_data():
-    # this is the augmentation configuration we will use for training
-    train_datagen = ImageDataGenerator(
-        rescale=1. / 255,
-        shear_range=0.2,
-        zoom_range=0.2,
-        horizontal_flip=True)
-
-    train_generator = train_datagen.flow_from_directory(
-        train_data_dir,
-        target_size=(img_width, img_height),
-        batch_size=batch_size,
-        class_mode='binary')
-
-    return train_generator
-
-'''
-Method which loads validation data
-@return : DirectoryIterator
-'''
-def load_validation_data():
-
-    # this is the augmentation configuration we will use for testing:
-    # only rescaling
-    test_datagen = ImageDataGenerator(rescale=1. / 255)
-
-    validation_generator = test_datagen.flow_from_directory(
-        validation_data_dir,
-        target_size=(img_width, img_height),
-        batch_size=batch_size,
-        class_mode='binary')
-
-    return validation_generator
-
-def main():
-
-    # update global variable and not local
-    global batch_size
-    global epochs   
-    global img_width
-    global img_height
-    global input_shape
-    global train_data_dir
-    global validation_data_dir
-    global nb_train_samples
-    global nb_validation_samples 
-
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:d:b:e:i", ["help", "output=", "directory=", "batch_size=", "epochs=", "img="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-            sys.exit()
-        elif o in ("-o", "--output"):
-            filename = a
-        elif o in ("-b", "--batch_size"):
-            batch_size = int(a)
-            print(batch_size)
-        elif o in ("-e", "--epochs"):
-            epochs = int(a)
-        elif o in ("-d", "--directory"):
-            directory = a
-        elif o in ("-i", "--img"):
-            img_height = int(a)
-            img_width = int(a)
-        else:
-            assert False, "unhandled option"
-
-    # 3 because we have 3 color canals
-    if K.image_data_format() == 'channels_first':
-        input_shape = (3, img_width, img_height)
-    else:
-        input_shape = (img_width, img_height, 3)
-
-    # configuration
-    with open('config.json') as json_data:
-        d = json.load(json_data)
-        train_data_dir = d['train_data_dir']
-        validation_data_dir = d['train_validation_dir']
-
-        try:
-            nb_train_samples = d[str(img_width)]['nb_train_samples']
-            nb_validation_samples = d[str(img_width)]['nb_validation_samples']
-        except:
-             print("--img parameter missing of invalid (--image_width xx --img_height xx)")
-             sys.exit(2)
-    
-    # load of model
-    model = generate_model()
-    model.summary()
-
-    if 'directory' in locals():
-        print('Your model information will be saved into %s...' % directory)
-
-    history = model.fit_generator(
-        load_train_data(),
-        steps_per_epoch=nb_train_samples // batch_size,
-        epochs=epochs,
-        validation_data=load_validation_data(),
-        validation_steps=nb_validation_samples // batch_size)
-
-    # if user needs output files
-    if(filename):
-
-        # update filename by folder
-        if(directory):
-            # create folder if necessary
-            if not os.path.exists(directory):
-                os.makedirs(directory)
-            filename = directory + "/" + filename
-
-        # save plot file history
-        tf_model_helper.save(history, filename)
-
-        plot_model(model, to_file=str(('%s.png' % filename)))
-        model.save_weights(str('%s.h5' % filename))
-
-
-if __name__ == "__main__":
-    main()

classification_cnn_keras_cross_validation.py

@@ -1,235 +0,0 @@
-'''This script goes along the blog post
-"Building powerful image classification models using very little data"
-from blog.keras.io.
-```
-data/
-    train/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-    validation/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-```
-'''
-import sys, os, getopt
-import json
-
-from keras.preprocessing.image import ImageDataGenerator
-from keras.models import Sequential
-from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D
-from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
-from keras import backend as K
-from keras.utils import plot_model
-
-
-# local functions import (metrics preprocessing)
-import preprocessing_functions
-
-##########################################
-# Global parameters (with default value) #
-##########################################
-img_width, img_height = 100, 100
-
-train_data_dir = 'data/train'
-validation_data_dir = 'data/validation'
-nb_train_samples = 7200
-nb_validation_samples = 3600
-epochs = 50
-batch_size = 16
-
-input_shape = (3, img_width, img_height)
-
-###########################################
-
-'''
-Method which returns model to train
-@return : DirectoryIterator
-'''
-def generate_model():
-    # create your model using this function
-    model = Sequential()
-    model.add(Conv2D(60, (2, 2), input_shape=input_shape, dilation_rate=1))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-
-    model.add(Conv2D(40, (2, 2), dilation_rate=1))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-
-    model.add(Conv2D(20, (2, 2), dilation_rate=1))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-
-    model.add(Flatten())
-
-    model.add(Dense(140))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.3))
-
-    model.add(Dense(120))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.3))
-
-    model.add(Dense(80))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.2))
-
-    model.add(Dense(40))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.2))
-
-    model.add(Dense(20))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.2))
-
-    model.add(Dense(1))
-    model.add(Activation('sigmoid'))
-
-    model.compile(loss='binary_crossentropy',
-                  optimizer='adam',
-                  metrics=['accuracy'])
-
-    return model
-
-def load_data():
-    # load your data using this function
-    # this is the augmentation configuration we will use for training
-    train_datagen = ImageDataGenerator(
-        rescale=1. / 255,
-        shear_range=0.2,
-        zoom_range=0.2,
-        horizontal_flip=True)
-
-    train_generator = train_datagen.flow_from_directory(
-        train_data_dir,
-        target_size=(img_width, img_height),
-        batch_size=batch_size,
-        class_mode='binary')
-
-    return train_generator
-
-def train_and_evaluate_model(model, data_train, data_test):
-
-    return model.fit_generator(
-        data_train,
-        steps_per_epoch=nb_train_samples // batch_size,
-        epochs=epochs,
-        shuffle=True,
-        validation_data=data_test,
-        validation_steps=nb_validation_samples // batch_size)
-
-def main():
-
-    # update global variable and not local
-    global batch_size
-    global epochs
-    global img_width
-    global img_height
-    global input_shape
-    global train_data_dir
-    global validation_data_dir
-    global nb_train_samples
-    global nb_validation_samples
-
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:d:b:e:i", ["help", "output=", "directory=", "batch_size=", "epochs=", "img="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-            sys.exit()
-        elif o in ("-o", "--output"):
-            filename = a
-        elif o in ("-b", "--batch_size"):
-            batch_size = int(a)
-        elif o in ("-e", "--epochs"):
-            epochs = int(a)
-        elif o in ("-d", "--directory"):
-            directory = a
-        elif o in ("-i", "--img"):
-            img_height = int(a)
-            img_width = int(a)
-        else:
-            assert False, "unhandled option"
-
-    # 3 because we have 3 color canals
-    if K.image_data_format() == 'channels_first':
-        input_shape = (3, img_width, img_height)
-    else:
-        input_shape = (img_width, img_height, 3)
-
-    # configuration
-    with open('config.json') as json_data:
-        d = json.load(json_data)
-        train_data_dir = d['train_data_dir']
-        validation_data_dir = d['train_validation_dir']
-
-        try:
-            nb_train_samples = d[str(img_width)]['nb_train_samples']
-            nb_validation_samples = d[str(img_width)]['nb_validation_samples']
-        except:
-             print("--img parameter missing of invalid (--image_width xx --img_height xx)")
-             sys.exit(2)
-
-
-    # load of model
-    model = generate_model()
-    model.summary()
-
-    data_generator = ImageDataGenerator(rescale=1./255, validation_split=0.33)
-
-    # check if possible to not do this thing each time
-    train_generator = data_generator.flow_from_directory(train_data_dir, target_size=(img_width, img_height), shuffle=True, seed=13,
-                                                         class_mode='binary', batch_size=batch_size, subset="training")
-
-    validation_generator = data_generator.flow_from_directory(train_data_dir, target_size=(img_width, img_height), shuffle=True, seed=13,
-                                                         class_mode='binary', batch_size=batch_size, subset="validation")
-
-    # now run model
-    history = train_and_evaluate_model(model, train_generator, validation_generator)
-
-    print("directory %s " % directory)
-    if(directory):
-        print('Your model information will be saved into %s...' % directory)
-    # if user needs output files
-    if(filename):
-
-        # update filename by folder
-        if(directory):
-            # create folder if necessary
-            if not os.path.exists(directory):
-                os.makedirs(directory)
-            filename = directory + "/" + filename
-
-        # save plot file history
-        # tf_model_helper.save(history, filename)
-
-        plot_model(model, to_file=str(('%s.png' % filename)))
-        model.save_weights(str('%s.h5' % filename))
-
-if __name__ == "__main__":
-    main()

classification_cnn_keras_svd.py

@@ -1,254 +0,0 @@
-'''This script goes along the blog post
-"Building powerful image classification models using very little data"
-from blog.keras.io.
-```
-data/
-    train/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-    validation/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-```
-'''
-import sys, os, getopt
-import json
-
-from keras.preprocessing.image import ImageDataGenerator
-from keras.models import Sequential
-from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D
-from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
-from keras.optimizers import Adam
-from keras.regularizers import l2
-from keras import backend as K
-from keras.utils import plot_model
-
-import tensorflow as tf
-import numpy as np
-
-from ipfml import tf_model_helper
-from ipfml import metrics
-
-# local functions import
-import preprocessing_functions
-
-##########################################
-# Global parameters (with default value) #
-##########################################
-img_width, img_height = 100, 100
-
-train_data_dir = 'data/train'
-validation_data_dir = 'data/validation'
-nb_train_samples = 7200
-nb_validation_samples = 3600
-epochs = 50
-batch_size = 16
-
-input_shape = (3, img_width, img_height)
-
-###########################################
-
-'''
-Method which returns model to train
-@return : DirectoryIterator
-'''
-def generate_model():
-
-    model = Sequential()
-
-    model.add(Conv2D(60, (2, 1), input_shape=input_shape))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(MaxPooling2D(pool_size=(2, 1)))
-
-    model.add(Conv2D(40, (2, 1)))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 1)))
-
-    model.add(Conv2D(30, (2, 1)))
-    model.add(Activation('relu'))
-    model.add(MaxPooling2D(pool_size=(2, 1)))
-
-    model.add(Flatten())
-    model.add(Dense(150, kernel_regularizer=l2(0.01)))
-    model.add(BatchNormalization())
-    model.add(Activation('relu'))
-    model.add(Dropout(0.2))
-
-    model.add(Dense(120, kernel_regularizer=l2(0.01)))
-    model.add(BatchNormalization())
-    model.add(Activation('relu'))
-    model.add(Dropout(0.2))
-
-    model.add(Dense(80, kernel_regularizer=l2(0.01)))
-    model.add(BatchNormalization())
-    model.add(Activation('relu'))
-    model.add(Dropout(0.2))
-
-    model.add(Dense(40, kernel_regularizer=l2(0.01)))
-    model.add(BatchNormalization())
-    model.add(Activation('relu'))
-    model.add(Dropout(0.2))
-
-    model.add(Dense(20, kernel_regularizer=l2(0.01)))
-    model.add(BatchNormalization())
-    model.add(Activation('relu'))
-    model.add(Dropout(0.1))
-
-    model.add(Dense(1))
-    model.add(Activation('sigmoid'))
-
-    model.compile(loss='binary_crossentropy',
-                  optimizer='rmsprop',
-                  metrics=['accuracy'])
-
-    return model
-
-'''
-Method which loads train data
-@return : DirectoryIterator
-'''
-def load_train_data():
-
-    # this is the augmentation configuration we will use for training
-    train_datagen = ImageDataGenerator(
-        #rescale=1. / 255,
-        #shear_range=0.2,
-        #zoom_range=0.2,
-        #horizontal_flip=True,
-        preprocessing_function=preprocessing_functions.get_s_model_data)
-
-    train_generator = train_datagen.flow_from_directory(
-        train_data_dir,
-        target_size=(img_width, img_height),
-        batch_size=batch_size,
-        class_mode='binary')
-
-    return train_generator
-
-'''
-Method which loads validation data
-@return : DirectoryIterator
-'''
-def load_validation_data():
-
-    # this is the augmentation configuration we will use for testing:
-    # only rescaling
-    test_datagen = ImageDataGenerator(
-        #rescale=1. / 255,
-        preprocessing_function=preprocessing_functions.get_s_model_data)
-
-    validation_generator = test_datagen.flow_from_directory(
-        validation_data_dir,
-        target_size=(img_width, img_height),
-        batch_size=batch_size,
-        class_mode='binary')
-
-    return validation_generator
-
-def main():
-
-    # update global variable and not local
-    global batch_size
-    global epochs   
-    global img_width
-    global img_height
-    global input_shape
-    global train_data_dir
-    global validation_data_dir
-    global nb_train_samples
-    global nb_validation_samples 
-
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:d:b:e:i", ["help", "output=", "directory=", "batch_size=", "epochs=", "img="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
-            sys.exit()
-        elif o in ("-o", "--output"):
-            filename = a
-        elif o in ("-b", "--batch_size"):
-            batch_size = int(a)
-        elif o in ("-e", "--epochs"):
-            epochs = int(a)
-        elif o in ("-d", "--directory"):
-            directory = a
-        elif o in ("-i", "--img"):
-            img_height = int(a)
-            img_width = int(a)
-        else:
-            assert False, "unhandled option"
-
-    # 3 because we have 3 color canals
-    if K.image_data_format() == 'channels_first':
-        input_shape = (3, img_width, img_height)
-    else:
-        input_shape = (img_width, img_height, 3)
-
-    # configuration
-    with open('config.json') as json_data:
-        d = json.load(json_data)
-        train_data_dir = d['train_data_dir']
-        validation_data_dir = d['train_validation_dir']
-
-        try:
-            nb_train_samples = d[str(img_width)]['nb_train_samples']
-            nb_validation_samples = d[str(img_width)]['nb_validation_samples']
-        except:
-             print("--img parameter missing of invalid (--image_width xx --img_height xx)")
-             sys.exit(2)
-
-
-    # load of model
-    model = generate_model()
-    model.summary()
-
-    if(directory):
-        print('Your model information will be saved into %s...' % directory)
-
-    history = model.fit_generator(
-        load_train_data(),
-        steps_per_epoch=nb_train_samples // batch_size,
-        epochs=epochs,
-        validation_data=load_validation_data(),
-        validation_steps=nb_validation_samples // batch_size)
-
-    # if user needs output files
-    if(filename):
-
-        # update filename by folder
-        if(directory):
-            # create folder if necessary
-            if not os.path.exists(directory):
-                os.makedirs(directory)
-            filename = directory + "/" + filename
-
-        # save plot file history
-        tf_model_helper.save(history, filename)
-
-        plot_model(model, to_file=str(('%s.png' % filename)), show_shapes=True)
-        model.save_weights(str('%s.h5' % filename))
-
-
-if __name__ == "__main__":
-    main()

classification_cnn_keras_svd_img.py

@@ -1,275 +0,0 @@
-'''This script goes along the blog post
-"Building powerful image classification models using very little data"
-from blog.keras.io.
-```
-data/
-    train/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-    validation/
-        final/
-            final001.png
-            final002.png
-            ...
-        noisy/
-            noisy001.png
-            noisy002.png
-            ...
-```
-'''
-import sys, os, getopt
-import json
-
-from keras.preprocessing.image import ImageDataGenerator
-from keras.models import Sequential
-from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, Cropping2D
-from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
-from keras.optimizers import Adam
-from keras.regularizers import l2
-from keras import backend as K
-from keras.utils import plot_model
-
-import tensorflow as tf
-import numpy as np
-
-import matplotlib.pyplot as plt
-
-# preprocessing of images
-from path import Path
-from PIL import Image
-import shutil
-import time
-
-# local functions import (metrics preprocessing)
-import preprocessing_functions
-
-##########################################
-# Global parameters (with default value) #
-#### ######################################
-img_width, img_height = 100, 100
-
-train_data_dir = 'data_svd_**img_size**/train'
-validation_data_dir = 'data_svd_**img_size**/validation'
-nb_train_samples = 7200
-nb_validation_samples = 3600
-epochs = 50
-batch_size = 16
-
-input_shape = (3, img_width, img_height)
-
-###########################################
-
-def init_directory(img_size, generate_data):
-
-    img_size_str = str(img_size)
-
-    svd_data_folder = str('data_svd_' + img_size_str)
-
-    if os.path.exists(svd_data_folder) and 'y' in generate_data:
-        print("Removing all previous data...")
-
-        shutil.rmtree(svd_data_folder)
-
-    if not os.path.exists(svd_data_folder):
-        print("Creating new data... Just take coffee... Or two...")
-        os.makedirs(str(train_data_dir.replace('**img_size**', img_size_str) + '/final'))
-        os.makedirs(str(train_data_dir.replace('**img_size**', img_size_str) + '/noisy'))
-
-        os.makedirs(str(validation_data_dir.replace('**img_size**', img_size_str) + '/final'))
-        os.makedirs(str(validation_data_dir.replace('**img_size**', img_size_str) + '/noisy'))
-
-        for f in Path('./data').walkfiles():
-            if 'png' in f:
-                img = Image.open(f)
-                new_img = preprocessing_functions.get_s_model_data_img(img)
-                new_img_path = f.replace('./data', str('./' + svd_data_folder))
-                new_img.save(new_img_path)
-                print(new_img_path)
-
-
-'''
-Method which returns model to train
-@return : DirectoryIterator
-'''
-def generate_model():
-
-    model = Sequential()
-
-    model.add(Cropping2D(cropping=((20, 20), (20, 20)), input_shape=input_shape))
-
-    model.add(Conv2D(50, (2, 2)))
-    model.add(Activation('relu'))
-    model.add(AveragePooling2D(pool_size=(2, 2)))
-
-    model.add(Flatten())
-
-    model.add(Dense(100, kernel_regularizer=l2(0.01)))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.2))
-
-    model.add(Dense(100, kernel_regularizer=l2(0.01)))
-    model.add(Activation('relu'))
-    model.add(BatchNormalization())
-    model.add(Dropout(0.5))
-
-    model.add(Dense(1))
-    model.add(Activation('sigmoid'))
-
-    model.compile(loss='binary_crossentropy',
-                  optimizer='rmsprop',
-                  metrics=['accuracy'])
-
-    return model
-
-'''
-Method which loads train data
-@return : DirectoryIterator
-'''
-def load_train_data():
-
-    # this is the augmentation configuration we will use for training
-    train_datagen = ImageDataGenerator(
-        rescale=1. / 255,
-        #shear_range=0.2,
-        #zoom_range=0.2,
-        #horizontal_flip=True,
-        #preprocessing_function=preprocessing_functions.get_s_model_data_img
-        )
-
-    train_generator = train_datagen.flow_from_directory(
-        train_data_dir,
-        target_size=(img_width, img_height),
-        batch_size=batch_size,
-        class_mode='binary')
-
-    return train_generator
-
-'''
-Method which loads validation data
-@return : DirectoryIterator
-'''
-def load_validation_data():
-
-    # this is the augmentation configuration we will use for testing:
-    # only rescaling
-    test_datagen = ImageDataGenerator(
-        rescale=1. / 255,
-        #preprocessing_function=preprocessing_functions.get_s_model_data_img
-        )
-
-    validation_generator = test_datagen.flow_from_directory(
-        validation_data_dir,
-        target_size=(img_width, img_height),
-        batch_size=batch_size,
-        class_mode='binary')
-
-    return validation_generator
-
-def main():
-
-    # update global variable and not local
-    global batch_size
-    global epochs
-    global input_shape
-    global train_data_dir
-    global validation_data_dir
-    global nb_train_samples
-    global nb_validation_samples
-
-    if len(sys.argv) <= 1:
-        print('Run with default parameters...')
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx --generate (y/n)')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:d:b:e:i:g", ["help", "output=", "directory=", "batch_size=", "epochs=", "img=", "generate="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx --generate (y/n)')
-        sys.exit(2)
-    for o, a in opts:
-        if o == "-h":
-            print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx --generate (y/n)')
-            sys.exit()
-        elif o in ("-o", "--output"):
-            filename = a
-        elif o in ("-b", "--batch_size"):
-            batch_size = int(a)
-        elif o in ("-e", "--epochs"):
-            epochs = int(a)
-        elif o in ("-d", "--directory"):
-            directory = a
-        elif o in ("-i", "--img"):
-            image_size = int(a)
-        elif o in ("-g", "--generate"):
-            generate_data = a
-        else:
-            assert False, "unhandled option"
-
-    # 3 because we have 3 color canals
-    if K.image_data_format() == 'channels_first':
-        input_shape = (3, img_width, img_height)
-    else:
-        input_shape = (img_width, img_height, 3)
-
-    img_str_size = str(image_size)
-    train_data_dir = str(train_data_dir.replace('**img_size**', img_str_size))
-    validation_data_dir = str(validation_data_dir.replace('**img_size**', img_str_size))
-
-    # configuration
-    with open('config.json') as json_data:
-        d = json.load(json_data)
-
-        try:
-            nb_train_samples = d[str(image_size)]['nb_train_samples']
-            nb_validation_samples = d[str(image_size)]['nb_validation_samples']
-        except:
-             print("--img parameter missing of invalid (--image_width xx --img_height xx)")
-             sys.exit(2)
-
-
-    init_directory(image_size, generate_data)
-    # load of model
-    model = generate_model()
-    model.summary()
-
-    if(directory):
-        print('Your model information will be saved into %s...' % directory)
-
-    history = model.fit_generator(
-        load_train_data(),
-        steps_per_epoch=nb_train_samples // batch_size,
-        epochs=epochs,
-        validation_data=load_validation_data(),
-        validation_steps=nb_validation_samples // batch_size)
-
-    # if user needs output files
-    if(filename):
-
-        # update filename by folder
-        if(directory):
-            # create folder if necessary
-            if not os.path.exists(directory):
-                os.makedirs(directory)
-            filename = directory + "/" + filename
-
-        fig_size = plt.rcParams["figure.figsize"]
-        fig_size[0] = 9
-        fig_size[1] = 9
-        plt.rcParams["figure.figsize"] = fig_size
-
-        # save plot file history
-        plot_info.save(history, filename)
-
-        plot_model(model, to_file=str(('%s.png' % filename)), show_shapes=True)
-        model.save_weights(str('%s.h5' % filename))
-
-
-if __name__ == "__main__":
-    main()

config.json

@@ -1,29 +0,0 @@
-{
-    "train_data_dir": "data/train",
-    "train_validation_dir": "data/validation",
-
-    "20":{    
-        "nb_train_samples": 115200,
-        "nb_validation_samples": 57600
-    },
-
-    "40":{    
-        "nb_train_samples": 57600,
-        "nb_validation_samples": 28800
-    },
-
-    "60":{    
-        "nb_train_samples": 12800,
-        "nb_validation_samples": 6400
-    },
-
-    "80":{    
-        "nb_train_samples": 7200,
-        "nb_validation_samples": 3600
-    },
-
-    "100":{    
-        "nb_train_samples": 4608,
-        "nb_validation_samples": 2304
-    }
-}

generate_dataset.py

@@ -1,81 +1,179 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Created on Fri Sep 14 21:02:42 2018
+Created on Wed Jun 19 11:47:42 2019
 
 @author: jbuisine
 """
 
-from __future__ import print_function
-import glob, image_slicer
-import sys, os, getopt
+import sys, os, argparse
+import numpy as np
+import random
+import time
+import json
+
 from PIL import Image
-import shutil
+from ipfml import processing, metrics, utils
+from skimage import color
+
+from modules.utils import config as cfg
+from modules.utils import data as dt
+
+from preprocessing_functions import svd_reconstruction
+
+# getting configuration information
+config_filename         = cfg.config_filename
+zone_folder             = cfg.zone_folder
+learned_folder          = cfg.learned_zones_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes values
+scenes_list             = cfg.scenes_names
+scenes_indexes          = cfg.scenes_indices
+choices                 = cfg.normalization_choices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+metric_choices          = cfg.metric_choices_labels
+output_data_folder      = cfg.output_data_folder
+
+generic_output_file_svd = '_random.csv'
+
+def generate_data_model(_scenes_list, _filename, _interval,  _metric, _scenes, _nb_zones = 4, _random=0):
+
+    output_train_filename = _filename + ".train"
+    output_test_filename = _filename + ".test"
+
+    if not '/' in output_train_filename:
+        raise Exception("Please select filename with directory path to save data. Example : data/dataset")
+
+    # create path if not exists
+    if not os.path.exists(output_data_folder):
+        os.makedirs(output_data_folder)
+
+    train_file_data = []
+    test_file_data  = []
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+    begin, end = _interval
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(_scenes_list):
+
+        scene_path = os.path.join(path, folder_scene)
+
+        zones_indices = zones
+
+        # shuffle list of zones (=> randomly choose zones)
+        # only in random mode
+        if _random:
+            random.shuffle(zones_indices)
 
-# show to create own dataset https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d
+         # store zones learned
+        learned_zones_indices = zones_indices[:_nb_zones]
 
-NUMBER_SUB_IMAGES = 100
+        # write into file
+        folder_learned_path = os.path.join(learned_folder, _filename.split('/')[1])
 
-def init_directory():
+        if not os.path.exists(folder_learned_path):
+            os.makedirs(folder_learned_path)
 
-    if os.path.exists('data'):
-        print("Removing all previous data...")
+        file_learned_path = os.path.join(folder_learned_path, folder_scene + '.csv')
 
-        shutil.rmtree('data')
+        with open(file_learned_path, 'w') as f:
+            for i in learned_zones_indices:
+                f.write(str(i) + ';')
 
-    if not os.path.exists('data'):
-        print("Creating new data...")
-        os.makedirs('data/train/final')
-        os.makedirs('data/train/noisy')
+        for id_zone, index_folder in enumerate(zones_indices):
 
-        os.makedirs('data/validation/final')
-        os.makedirs('data/validation/noisy')
+            index_str = str(index_folder)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+            
+            current_zone_folder = "zone" + index_str
+            zone_path = os.path.join(scene_path, current_zone_folder)
 
-def create_images(folder, output_folder):
-    images_path = glob.glob(folder + "/*.png")
+            # custom path for metric
+            metric_path = os.path.join(zone_path, _metric)
 
-    for img in images_path:
-        image_name = img.replace(folder, '').replace('/', '')
-        tiles = image_slicer.slice(img, NUMBER_SUB_IMAGES, save = False)
-        image_slicer.save_tiles(tiles, directory=output_folder, prefix='part_'+image_name)
+            # custom path for interval of reconstruction and metric
+            metric_interval_path = os.path.join(metric_path, str(begin) + "_" + str(end))
 
-def generate_dataset():
-    create_images('img_train/final', 'data/train/final')
-    create_images('img_train/noisy', 'data/train/noisy')
-    create_images('img_validation/final', 'data/validation/final')
-    create_images('img_validation/noisy', 'data/validation/noisy')
+            for label in os.listdir(metric_interval_path):
+                label_path = os.path.join(metric_interval_path, label)
+
+                images = os.listdir(label_path)
+
+                for img in images:
+                    img_path = os.path.join(label_path, img)
+
+                    if label == cfg.noisy_folder:
+                        line = '1;' + img_path + '\n'
+                    else:
+                        line = '0;' + img_path + '\n'
+
+                    if id_zone < _nb_zones and folder_scene in _scenes:
+                        train_file_data.append(line)
+                    else:
+                        test_file_data.append(line)
+
+    train_file = open(output_train_filename, 'w')
+    test_file = open(output_test_filename, 'w')
+
+    random.shuffle(train_file_data)
+    random.shuffle(test_file_data)
+
+    for line in train_file_data:
+        train_file.write(line)
+
+    for line in test_file_data:
+        test_file.write(line)
+
+    train_file.close()
+    test_file.close()
 
 def main():
 
-    global NUMBER_SUB_IMAGES
-
-    if len(sys.argv) <= 1:
-        print('Please specify nb sub image per image parameter (use -h if you want to know values)...')
-        print('generate_dataset.py --nb xxxx')
-        sys.exit(2)
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "h:n", ["help", "nb="])
-    except getopt.GetoptError:
-        # print help information and exit:
-        print('generate_dataset.py --nb xxxx')
-        sys.exit(2)
-    for o, a in opts:
-
-        if o == "--help":
-            print('generate_dataset.py --nb xxxx')
-            print('20x20 : 1600')
-            print('40x40 : 400')
-            print('60x60 : 178 (approximately)')
-            print('80x80 : 100')
-            print('100x100 : 64')
-            sys.exit()
-        elif o == '--nb':
-            NUMBER_SUB_IMAGES = int(a)
-
-    init_directory()
+    parser = argparse.ArgumentParser(description="Compute specific dataset for model using of metric")
+
+    parser.add_argument('--output', type=str, help='output file name desired (.train and .test)')
+    parser.add_argument('--metric', type=str, 
+                                    help="metric choice in order to compute data (use 'all' if all metrics are needed)", 
+                                    choices=metric_choices,
+                                    required=True)
+    parser.add_argument('--interval', type=str, help="interval choice if needed by the compression method", default='"100, 200"')
+    parser.add_argument('--scenes', type=str, help='List of scenes to use for training data')
+    parser.add_argument('--nb_zones', type=int, help='Number of zones to use for training data set', choices=list(range(1, 17)))
+    parser.add_argument('--renderer', type=str, help='Renderer choice in order to limit scenes used', choices=cfg.renderer_choices, default='all')
+    parser.add_argument('--random', type=int, help='Data will be randomly filled or not', choices=[0, 1])
+
+    args = parser.parse_args()
+
+    p_filename = args.output
+    p_metric   = args.metric
+    p_interval = list(map(int, args.interval.split(',')))
+    p_scenes   = args.scenes.split(',')
+    p_nb_zones = args.nb_zones
+    p_renderer = args.renderer
+    p_random   = args.random
+
+        # list all possibles choices of renderer
+    scenes_list = dt.get_renderer_scenes_names(p_renderer)
+    scenes_indices = dt.get_renderer_scenes_indices(p_renderer)
+
+    # getting scenes from indexes user selection
+    scenes_selected = []
+
+    for scene_id in p_scenes:
+        index = scenes_indices.index(scene_id.strip())
+        scenes_selected.append(scenes_list[index])
 
     # create database using img folder (generate first time only)
-    generate_dataset()
+    generate_data_model(scenes_list, p_filename, p_interval,  p_metric, scenes_selected, p_nb_zones, p_random)
+
 
 if __name__== "__main__":
     main()

generate_reconstructed_data.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jun 19 11:47:42 2019
+
+@author: jbuisine
+"""
+
+import sys, os, argparse
+import numpy as np
+import random
+import time
+import json
+
+from PIL import Image
+from ipfml import processing, metrics, utils
+from skimage import color
+
+from modules.utils import config as cfg
+from preprocessing_functions import svd_reconstruction
+
+# getting configuration information
+config_filename         = cfg.config_filename
+zone_folder             = cfg.zone_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes values
+scenes_list             = cfg.scenes_names
+scenes_indexes          = cfg.scenes_indices
+choices                 = cfg.normalization_choices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+metric_choices          = cfg.metric_choices_labels
+output_data_folder      = cfg.output_data_folder
+
+generic_output_file_svd = '_random.csv'
+
+def generate_data_svd(data_type, interval):
+    """
+    @brief Method which generates all .csv files from scenes
+    @param data_type,  metric choice
+    @param interval, interval choice used by reconstructed method
+    @return nothing
+    """
+
+    scenes = os.listdir(path)
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+    begin, end = interval
+
+    # go ahead each scenes
+    for id_scene, folder_scene in enumerate(scenes):
+
+        print(folder_scene)
+        scene_path = os.path.join(path, folder_scene)
+
+        config_file_path = os.path.join(scene_path, config_filename)
+
+        with open(config_file_path, "r") as config_file:
+            last_image_name = config_file.readline().strip()
+            prefix_image_name = config_file.readline().strip()
+            start_index_image = config_file.readline().strip()
+            end_index_image = config_file.readline().strip()
+            step_counter = int(config_file.readline().strip())
+
+        # construct each zones folder name
+        zones_folder = []
+        metrics_folder = []
+        zones_threshold = []
+
+        # get zones list info
+        for index in zones:
+            index_str = str(index)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+
+            current_zone = "zone"+index_str
+            zones_folder.append(current_zone)
+            zone_path = os.path.join(scene_path, current_zone)
+
+            with open(os.path.join(zone_path, cfg.seuil_expe_filename)) as f:
+                zones_threshold.append(int(f.readline()))
+
+            # custom path for metric
+            metric_path = os.path.join(zone_path, data_type)
+
+            if not os.path.exists(metric_path):
+                os.makedirs(metric_path)
+
+            # custom path for interval of reconstruction and metric
+            metric_interval_path = os.path.join(metric_path, str(begin) + "_" + str(end))
+            metrics_folder.append(metric_interval_path)
+
+            if not os.path.exists(metric_interval_path):
+                os.makedirs(metric_interval_path)
+
+            # create for each zone the labels folder
+            labels = [cfg.not_noisy_folder, cfg.noisy_folder]
+
+            for label in labels:
+                label_folder = os.path.join(metric_interval_path, label)
+
+                if not os.path.exists(label_folder):
+                    os.makedirs(label_folder)
+
+        
+
+        current_counter_index = int(start_index_image)
+        end_counter_index = int(end_index_image)
+
+        # for each images
+        while(current_counter_index <= end_counter_index):
+
+            current_counter_index_str = str(current_counter_index)
+
+            while len(start_index_image) > len(current_counter_index_str):
+                current_counter_index_str = "0" + current_counter_index_str
+
+            img_path = os.path.join(scene_path, prefix_image_name + current_counter_index_str + ".png")
+
+            current_img = Image.open(img_path)
+            img_blocks = processing.divide_in_blocks(current_img, (200, 200))
+
+            for id_block, block in enumerate(img_blocks):
+
+                ##########################
+                # Image computation part #
+                ##########################
+                output_block = svd_reconstruction(block, [begin, end])
+                output_block = np.array(output_block, 'uint8')
+                
+                # current output image
+                output_block_img = Image.fromarray(output_block)
+
+                label_path = metrics_folder[id_block]
+
+                # get label folder for block
+                if current_counter_index > zones_threshold[id_block]:
+                    label_path = os.path.join(label_path, cfg.not_noisy_folder)
+                else:
+                    label_path = os.path.join(label_path, cfg.noisy_folder)
+
+                rotations = [0, 90, 180, 270]
+
+                # rotate image to increase dataset size
+                for rotation in rotations:
+                    rotated_output_img = output_block_img.rotate(rotation)
+
+                    output_reconstructed_filename = img_path.split('/')[-1].replace('.png', '') + '_' + zones_folder[id_block]
+                    output_reconstructed_filename = output_reconstructed_filename + '_' + str(rotation) + '.png'
+                    output_reconstructed_path = os.path.join(label_path, output_reconstructed_filename)
+
+                    rotated_output_img.save(output_reconstructed_path)
+
+
+            start_index_image_int = int(start_index_image)
+            print(data_type + "_" + folder_scene + " - " + "{0:.2f}".format((current_counter_index - start_index_image_int) / (end_counter_index - start_index_image_int)* 100.) + "%")
+            sys.stdout.write("\033[F")
+
+            current_counter_index += step_counter
+
+
+        print('\n')
+
+    print("%s_%s : end of data generation\n" % (data_type, interval))
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Compute and prepare data of metric of all scenes using specific interval if necessary")
+
+    parser.add_argument('--metric', type=str, 
+                                    help="metric choice in order to compute data (use 'all' if all metrics are needed)", 
+                                    choices=metric_choices,
+                                    required=True)
+
+    parser.add_argument('--interval', type=str, 
+                                    help="interval choice if needed by the compression method", 
+                                    default='"100, 200"')
+
+    args = parser.parse_args()
+
+    p_metric   = args.metric
+    p_interval = list(map(int, args.interval.split(',')))
+
+    # generate all or specific metric data
+    if p_metric == 'all':
+        for m in metric_choices:
+            generate_data_svd(m, p_interval)
+    else:
+        generate_data_svd(p_metric, p_interval)
+
+if __name__== "__main__":
+    main()