Parcourir la source

Merge branch 'release/v0.0.2'

Jérôme BUISINE il y a 5 ans
Parent
commit
6fb9df52df

+ 3 - 1
.gitignore

@@ -2,4 +2,6 @@
 *.png
 
 data
-saved_models
+saved_models
+
+__pycache__

+ 7 - 7
make_dataset.py

@@ -9,9 +9,9 @@ def compute_files(_n, _each_row, _each_column):
     """
     Read all folders and files of scenes in order to compute output dataset
     """
-    
+
     output_dataset_filename = cfg.output_file_prefix + _n + '_column_' + _each_column + '_row_' + _each_row + '.csv'
-    
+
     output_dataset_filename = os.path.join(cfg.output_data_folder, output_dataset_filename)
 
     if not os.path.exists(cfg.output_data_folder):
@@ -27,11 +27,11 @@ def compute_files(_n, _each_row, _each_column):
     scenes = [s for s in scenes if s not in cfg.folder_and_files_filtered]
     scenes = [s for s in scenes if '.csv' not in s] # do not keep generated .csv file
 
-    print(scenes)
+    # print(scenes)
 
     counter = 0
     number_of_elements = len(scenes) * cfg.number_of_rows * cfg.number_of_columns
-    print(number_of_elements, ' to manage')
+    #print(number_of_elements, ' to manage')
 
     for scene in scenes:
 
@@ -39,7 +39,7 @@ def compute_files(_n, _each_row, _each_column):
         columns_folder = os.listdir(scene_path)
 
         for id_column, column in enumerate(columns_folder):
-            
+
             if id_column % int(_each_column) == 0 :
 
                 folder_path = os.path.join(scene_path, column)
@@ -47,7 +47,7 @@ def compute_files(_n, _each_row, _each_column):
                 pixel_files_list = os.listdir(folder_path)
 
                 for id_row, pixel_file in enumerate(pixel_files_list):
-                    
+
                     if id_row % int(_each_row) == 0:
                         pixel_file_path = os.path.join(folder_path, pixel_file)
 
@@ -64,7 +64,7 @@ def compute_files(_n, _each_row, _each_column):
 
                             for val in pixel_values:
                                 saved_row += ';' + str(val)
-                            
+
                             saved_row += '\n'
 
                         # store mean and pixel values into .csv row

+ 11 - 1
models_info/models_comparisons.csv

@@ -1 +1,11 @@
-10_column_8_row_7_SGD.joblib;0.9708138110211402;
+model_name; number_samples; coeff_of_determination;
+10_column_7_row_7_SGD;60236;0.967103608014;
+10_column_8_row_7_SGD;52096;0.9668493240289788;
+10_column_9_row_7_SGD;46398;0.9657893418797697;
+10_column_10_row_7_SGD;42328;0.9662073687839958;
+10_column_7_row_8_SGD;52096;0.9678400765943843;
+10_column_8_row_8_SGD;45056;0.9717562796162227;
+10_column_9_row_8_SGD;40128;0.9690779553770827;
+10_column_10_row_8_SGD;36608;0.9696013575399688;
+10_column_7_row_9_SGD;46398;0.9698917329738924;
+10_column_8_row_9_SGD;40128;0.9680105640513834;

BIN
modules/__pycache__/__init__.cpython-36.pyc


BIN
modules/__pycache__/config.cpython-36.pyc


BIN
modules/__pycache__/metrics.cpython-36.pyc


+ 3 - 1
modules/config.py

@@ -2,6 +2,7 @@ output_data_folder              = "data"
 folder_scenes_path              = ".."
 models_information_folder       = 'models_info'
 saved_models_folder             = 'saved_models'
+reconstructed_folder            = 'reconstructed'
 
 output_file_prefix              = "dataset_"
 folder_and_files_filtered       = ["analyse", "make_dataset.py", ".vscode"]
@@ -11,4 +12,5 @@ number_of_columns               = 512
 
 kind_of_models                  = ["SGD", "Ridge", "SVR"]
 
-global_result_filepath          = "models_info/models_comparisons.csv"
+global_result_filepath          = "models_info/models_comparisons.csv"
+scenes_list                     = ['Exterieur01', 'Boulanger', 'CornellBoxNonVideTextureArcade', 'CornellBoxVide', 'Bar1', 'CornellBoxNonVideTextureDegrade', 'CornellBoxNonVideTextureDamier', 'CornellBoxVideTextureDamier', 'CornellBoxNonVide', 'Sponza1', 'Bureau1_cam2']

+ 87 - 0
reconstruct.py

@@ -0,0 +1,87 @@
+import numpy as np
+import pandas as pd
+
+import os, sys, argparse
+
+from sklearn import linear_model
+from sklearn import svm
+from sklearn.utils import shuffle
+
+import modules.config as cfg
+import modules.metrics as metrics
+
+from joblib import dump, load
+
+from PIL import Image
+
+def reconstruct(_scene_name, _model_path, _n):
+    
+    # construct the empty output image
+    output_image = np.empty([cfg.number_of_rows, cfg.number_of_columns])
+
+    # load the trained model
+    clf = load(_model_path)
+
+    # load scene and its `n` first pixel value data
+    scene_path = os.path.join(cfg.folder_scenes_path, _scene_name)
+
+    columns_folder = os.listdir(scene_path)
+    for id_column, column in enumerate(columns_folder):
+
+        folder_path = os.path.join(scene_path, column)
+        pixel_files_list = os.listdir(folder_path)
+
+        pixels = []
+
+        for id_row, pixel_file in enumerate(pixel_files_list):
+            
+            pixel_file_path = os.path.join(folder_path, pixel_file)
+            
+            with open(pixel_file_path, 'r') as f:
+
+                # predict the expected pixel value
+                lines = [float(l)/255. for l in f.readlines()]
+                pixel_values = lines[0:int(_n)]
+                pixels.append(pixel_values)
+
+        # predict column pixels and fill image column by column
+        pixels_predicted = clf.predict(pixels)
+        output_image[id_column] = pixels_predicted*255.
+
+        print("{0:.2f}%".format(id_column / cfg.number_of_columns * 100))
+        sys.stdout.write("\033[F")
+
+    return output_image
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Train model and saved it")
+
+    parser.add_argument('--scene', type=str, help='Scene name to reconstruct', choices=cfg.scenes_list)
+    parser.add_argument('--model_path', type=str, help='Model file path')
+    parser.add_argument('--n', type=str, help='Number of pixel values approximated to keep')
+    parser.add_argument('--image_name', type=str, help="The ouput image name")
+
+    args = parser.parse_args()
+
+    param_scene_name = args.scene
+    param_n = args.n
+    param_model_path = args.model_path
+    param_image_name = args.image_name
+
+    # get default value of `n` param
+    if not param_n:
+        param_n = param_model_path.split('_')[0]
+
+    output_image = reconstruct(param_scene_name, param_model_path, param_n)
+
+    if not os.path.exists(cfg.reconstructed_folder):
+        os.makedirs(cfg.reconstructed_folder)
+
+    image_path = os.path.join(cfg.reconstructed_folder, param_image_name)
+
+    img = Image.fromarray(np.uint8(output_image))
+    img.save(image_path)
+
+if __name__== "__main__":
+    main()

+ 18 - 9
run.sh

@@ -10,18 +10,27 @@ if [ "${erased}" == "Y" ]; then
     touch ${file_path}
 
     # add of header
-    echo 'model_name; coeff_of_determination;' >> ${file_path}
+    echo 'model_name; number_of_approximations; coeff_of_determination;' >> ${file_path}
 fi
 
-for model in {"SGD","Ridge","SVR"}; do
-    for row in {7,8,9,10}; do
-        for column in {7,8,9,10}; do
+for n in {10,15,20,25,30}; do
+    for model in {"SGD","Ridge","SVR"}; do
+        for row in {7,8,9,10}; do
+            for column in {7,8,9,10}; do
 
-            # Run creation of dataset and train model
-            DATASET_NAME="data/dataset_10_column_${column}_row_${row}.csv"
+                # Run creation of dataset and train model
+                DATASET_NAME="data/dataset_${n}_column_${column}_row_${row}.csv"
+                MODEL_NAME="${n}_column_${column}_row_${row}_${model}"
 
-            python make_dataset.py --n 10 --each_row ${row} --each_column ${column}
-            python train_model.py --data ${DATASET_NAME} --model ${model}
+                if ! grep -q "${MODEL_NAME}" "${file_path}"; then
+                    echo "Run computation for model ${MODEL_NAME}"
+
+                    python make_dataset.py --n ${n} --each_row ${row} --each_column ${column}
+                    python train_model.py --data ${DATASET_NAME} --model ${model}
+                else
+                    echo "${MODEL_NAME} results already computed.."
+                fi
+            done
         done
     done
-done
+done

+ 6 - 6
train_model.py

@@ -46,22 +46,22 @@ def train(_data_file, _model_name):
 
     print("Predicted coefficient of determination for ", _model_name, " : ", coeff)
 
-    # save the trained model, so check if saved folder exists 
+    # save the trained model, so check if saved folder exists
     if not os.path.exists(cfg.saved_models_folder):
         os.makedirs(cfg.saved_models_folder)
 
     # compute model filename
-    model_filename = _data_file.split('/')[-1].replace(cfg.output_file_prefix, '').replace('.csv', '') 
+    model_filename = _data_file.split('/')[-1].replace(cfg.output_file_prefix, '').replace('.csv', '')
     model_filename = model_filename + '_' + _model_name + '.joblib'
 
     model_file_path = os.path.join(cfg.saved_models_folder, model_filename)
     print("Model will be save into `", model_file_path, '`')
-    
+
     dump(clf, model_file_path)
 
     # save score into global_result.csv file
-    with open(cfg.global_result_filepath, "w") as f:
-       f.write(model_filename + ';' + str(coeff) + ';\n')
+    with open(cfg.global_result_filepath, "a") as f:
+       f.write(model_filename.replace('.joblib', '') + ';' + str(len(y)) + ';' + str(coeff) + ';\n')
 
 def main():
 
@@ -69,7 +69,7 @@ def main():
 
     parser.add_argument('--data', type=str, help='Filename of dataset')
     parser.add_argument('--model', type=str, help='Kind of model expected', choices=cfg.kind_of_models)
-    
+
     args = parser.parse_args()
 
     param_data_file = args.data