Update of generate dataset and add of config file

README.md

@@ -25,9 +25,11 @@ Note that the image input size need to change in you used specific size for your
 
 After your built your neural network in classification_cnn_keras.py, you just have to run it :
 ```
-python classification_cnn_keras.py
+classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx (or --image_width xx --img_height xx)
 ```
 
+A config file in json is available and keeps in memory all image sizes available.
+
 ## Modules
 
 This project contains modules :

classification_cnn_keras.py

@@ -24,6 +24,7 @@ data/
 ```
 '''
 import sys, os, getopt
+import json
 
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
@@ -35,7 +36,9 @@ from keras.utils import plot_model
 from modules.model_helper import plot_info
 
 
-# dimensions of our images.
+##########################################
+# Global parameters (with default value) #
+##########################################
 img_width, img_height = 100, 100
 
 train_data_dir = 'data/train'
@@ -43,12 +46,11 @@ validation_data_dir = 'data/validation'
 nb_train_samples = 7200
 nb_validation_samples = 3600
 epochs = 50
-batch_size = 30
+batch_size = 16
 
-if K.image_data_format() == 'channels_first':
-    input_shape = (3, img_width, img_height)
-else:
-    input_shape = (img_width, img_height, 3)
+input_shape = (3, img_width, img_height)
+
+###########################################
 
 '''
 Method which returns model to train
@@ -131,22 +133,30 @@ def load_validation_data():
 
 def main():
 
+    # update global variable and not local
     global batch_size
-    global epochs
+    global epochs   
+    global img_width
+    global img_height
+    global input_shape
+    global train_data_dir
+    global validation_data_dir
+    global nb_train_samples
+    global nb_validation_samples 
 
     if len(sys.argv) <= 1:
-        print('No output file defined...')
-        print('classification_cnn_keras_svd.py --output xxxxx')
+        print('Run with default parameters...')
+        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
         sys.exit(2)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:b:e:d", ["help", "directory=", "output=", "batch_size=", "epochs="])
+        opts, args = getopt.getopt(sys.argv[1:], "ho:d:b:e:i", ["help", "output=", "directory=", "batch_size=", "epochs=", "img="])
     except getopt.GetoptError:
         # print help information and exit:
-        print('classification_cnn_keras_svd.py --output xxxxx')
+        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
         sys.exit(2)
     for o, a in opts:
         if o == "-h":
-            print('classification_cnn_keras_svd.py --output xxxxx')
+            print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
             sys.exit()
         elif o in ("-o", "--output"):
             filename = a
@@ -156,15 +166,36 @@ def main():
             epochs = int(a)
         elif o in ("-d", "--directory"):
             directory = a
+        elif o in ("-i", "--img"):
+            img_height = int(a)
+            img_width = int(a)
         else:
             assert False, "unhandled option"
 
-
+    # 3 because we have 3 color canals
+    if K.image_data_format() == 'channels_first':
+        input_shape = (3, img_width, img_height)
+    else:
+        input_shape = (img_width, img_height, 3)
+
+    # configuration
+    with open('config.json') as json_data:
+        d = json.load(json_data)
+        train_data_dir = d['train_data_dir']
+        validation_data_dir = d['train_validation_dir']
+
+        try:
+            nb_train_samples = d[str(img_width)]['nb_train_samples']
+            nb_validation_samples = d[str(img_width)]['nb_validation_samples']
+        except:
+             print("--img parameter missing of invalid (--image_width xx --img_height xx)")
+             sys.exit(2)
+    
     # load of model
     model = generate_model()
     model.summary()
 
-    if(directory):
+    if 'directory' in locals():
         print('Your model information will be saved into %s...' % directory)
 
     history = model.fit_generator(

classification_cnn_keras_cross_validation.py

@@ -24,6 +24,7 @@ data/
 ```
 '''
 import sys, os, getopt
+import json
 
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
@@ -34,7 +35,9 @@ from keras.utils import plot_model
 
 from modules.model_helper import plot_info
 
-# dimensions of our images.
+##########################################
+# Global parameters (with default value) #
+##########################################
 img_width, img_height = 100, 100
 
 train_data_dir = 'data/train'
@@ -44,12 +47,9 @@ nb_validation_samples = 3600
 epochs = 50
 batch_size = 16
 
-if K.image_data_format() == 'channels_first':
-    input_shape = (3, img_width, img_height)
-else:
-    input_shape = (img_width, img_height, 3)
-
+input_shape = (3, img_width, img_height)
 
+###########################################
 
 '''
 Method which returns model to train
@@ -134,22 +134,30 @@ def train_and_evaluate_model(model, data_train, data_test):
 
 def main():
 
+    # update global variable and not local
     global batch_size
-    global epochs
+    global epochs   
+    global img_width
+    global img_height
+    global input_shape
+    global train_data_dir
+    global validation_data_dir
+    global nb_train_samples
+    global nb_validation_samples 
 
     if len(sys.argv) <= 1:
-        print('No output file defined...')
-        print('classification_cnn_keras_svd.py --output xxxxx')
+        print('Run with default parameters...')
+        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
         sys.exit(2)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:b:e:d", ["help", "directory=", "output=", "batch_size=", "epochs="])
+        opts, args = getopt.getopt(sys.argv[1:], "ho:d:b:e:i", ["help", "output=", "directory=", "batch_size=", "epochs=", "img="])
     except getopt.GetoptError:
         # print help information and exit:
-        print('classification_cnn_keras_svd.py --output xxxxx')
+        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
         sys.exit(2)
     for o, a in opts:
         if o == "-h":
-            print('classification_cnn_keras_svd.py --output xxxxx')
+            print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
             sys.exit()
         elif o in ("-o", "--output"):
             filename = a
@@ -159,16 +167,36 @@ def main():
             epochs = int(a)
         elif o in ("-d", "--directory"):
             directory = a
+        elif o in ("-i", "--img"):
+            img_height = int(a)
+            img_width = int(a)
         else:
             assert False, "unhandled option"
 
+    # 3 because we have 3 color canals
+    if K.image_data_format() == 'channels_first':
+        input_shape = (3, img_width, img_height)
+    else:
+        input_shape = (img_width, img_height, 3)
+
+    # configuration
+    with open('config.json') as json_data:
+        d = json.load(json_data)
+        train_data_dir = d['train_data_dir']
+        validation_data_dir = d['train_validation_dir']
+
+        try:
+            nb_train_samples = d[str(img_width)]['nb_train_samples']
+            nb_validation_samples = d[str(img_width)]['nb_validation_samples']
+        except:
+             print("--img parameter missing of invalid (--image_width xx --img_height xx)")
+             sys.exit(2)
+
 
     # load of model
     model = generate_model()
     model.summary()
 
-    n_folds = 10
-
     data_generator = ImageDataGenerator(rescale=1./255, validation_split=0.33)
 
     # check if possible to not do this thing each time

classification_cnn_keras_svd.py

@@ -24,6 +24,7 @@ data/
 ```
 '''
 import sys, os, getopt
+import json
 
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
@@ -42,22 +43,21 @@ import numpy as np
 from modules.model_helper import plot_info
 from modules.image_metrics import svd_metric
 
-
-# configuration
-# dimensions of our images.
-img_width, img_height = 100, 1
+##########################################
+# Global parameters (with default value) #
+##########################################
+img_width, img_height = 100, 100
 
 train_data_dir = 'data/train'
 validation_data_dir = 'data/validation'
 nb_train_samples = 7200
 nb_validation_samples = 3600
-epochs = 200
-batch_size = 30
+epochs = 50
+batch_size = 16
+
+input_shape = (3, img_width, img_height)
 
-if K.image_data_format() == 'channels_first':
-    input_shape = (3, img_width, img_height)
-else:
-    input_shape = (img_width, img_height, 3)
+###########################################
 
 '''
 Method which returns model to train
@@ -173,22 +173,30 @@ def load_validation_data():
 
 def main():
 
+    # update global variable and not local
     global batch_size
-    global epochs
+    global epochs   
+    global img_width
+    global img_height
+    global input_shape
+    global train_data_dir
+    global validation_data_dir
+    global nb_train_samples
+    global nb_validation_samples 
 
     if len(sys.argv) <= 1:
-        print('No output file defined...')
-        print('classification_cnn_keras_svd.py --output xxxxx')
+        print('Run with default parameters...')
+        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
         sys.exit(2)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "ho:b:e:d", ["help", "directory=", "output=", "batch_size=", "epochs="])
+        opts, args = getopt.getopt(sys.argv[1:], "ho:d:b:e:i", ["help", "output=", "directory=", "batch_size=", "epochs=", "img="])
     except getopt.GetoptError:
         # print help information and exit:
-        print('classification_cnn_keras_svd.py --output xxxxx')
+        print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
         sys.exit(2)
     for o, a in opts:
         if o == "-h":
-            print('classification_cnn_keras_svd.py --output xxxxx')
+            print('classification_cnn_keras_svd.py --directory xxxx --output xxxxx --batch_size xx --epochs xx --img xx')
             sys.exit()
         elif o in ("-o", "--output"):
             filename = a
@@ -198,9 +206,31 @@ def main():
             epochs = int(a)
         elif o in ("-d", "--directory"):
             directory = a
+        elif o in ("-i", "--img"):
+            img_height = int(a)
+            img_width = int(a)
         else:
             assert False, "unhandled option"
 
+    # 3 because we have 3 color canals
+    if K.image_data_format() == 'channels_first':
+        input_shape = (3, img_width, img_height)
+    else:
+        input_shape = (img_width, img_height, 3)
+
+    # configuration
+    with open('config.json') as json_data:
+        d = json.load(json_data)
+        train_data_dir = d['train_data_dir']
+        validation_data_dir = d['train_validation_dir']
+
+        try:
+            nb_train_samples = d[str(img_width)]['nb_train_samples']
+            nb_validation_samples = d[str(img_width)]['nb_validation_samples']
+        except:
+             print("--img parameter missing of invalid (--image_width xx --img_height xx)")
+             sys.exit(2)
+
 
     # load of model
     model = generate_model()

config.json

@@ -0,0 +1,29 @@
+{
+    "train_data_dir": "data/train",
+    "train_validation_dir": "data/validation",
+
+    "20":{    
+        "nb_train_samples": 115200,
+        "nb_validation_samples": 57600
+    },
+
+    "40":{    
+        "nb_train_samples": 57600,
+        "nb_validation_samples": 28800
+    },
+
+    "60":{    
+        "nb_train_samples": 12800,
+        "nb_validation_samples": 6400
+    },
+
+    "80":{    
+        "nb_train_samples": 7200,
+        "nb_validation_samples": 3600
+    },
+
+    "100":{    
+        "nb_train_samples": 4608,
+        "nb_validation_samples": 2304
+    }
+}

generate_dataset.py

@@ -7,21 +7,30 @@ Created on Fri Sep 14 21:02:42 2018
 """
 
 from __future__ import print_function
-import os, glob, image_slicer
+import glob, image_slicer
+import sys, os, getopt
 from PIL import Image
+import shutil
 
 # show to create own dataset https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d
 
 NUMBER_SUB_IMAGES = 100
 
 def init_directory():
-    if not os.path.exists('data'):
 
+    if os.path.exists('data'):
+        print("Removing all previous data...")
+
+        shutil.rmtree('data')
+
+    if not os.path.exists('data'):
+        print("Creating new data...")
         os.makedirs('data/train/final')
         os.makedirs('data/train/noisy')
 
         os.makedirs('data/validation/final')
         os.makedirs('data/validation/noisy')
+       
 
 def create_images(folder, output_folder):
     images_path = glob.glob(folder + "/*.png")
@@ -39,6 +48,30 @@ def generate_dataset():
 
 def main():
 
+    global NUMBER_SUB_IMAGES
+
+    if len(sys.argv) <= 1:
+        print('Please specify nb sub image per image parameter (use -h if you want to know values)...')
+        print('generate_dataset.py --nb xxxx')
+        sys.exit(2)
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hn", ["help", "nb="])
+    except getopt.GetoptError:
+        # print help information and exit:
+        print('generate_dataset.py --nb xxxx')
+        sys.exit(2)
+    for o, a in opts:
+        if o == "-h":
+            print('generate_dataset.py --nb xxxx')
+            print('20x20 : 1600')
+            print('40x40 : 400')
+            print('60x60 : 178 (approximately)')
+            print('80x80 : 100')
+            print('100x100 : 64')
+            sys.exit()
+        elif o == '-n':
+            NUMBER_SUB_IMAGES = int(a)
+
     init_directory()
 
     # create database using img folder (generate first time only)