Update of scripts and add of utils module

.gitignore

@@ -107,4 +107,5 @@ venv.bak/
 # project data
 expe_data
 dataset
+extracted_data
 .vscode

custom_config.py

@@ -7,6 +7,8 @@ context_vars = vars()
 data_expe_folder                = 'data_expe'
 data_augmented_filename         = 'augmented_dataset.csv'
 
+extracted_data_folder           = 'extracted_data'
+
 # variables
 image_scene_size                = (800, 800)
 image_zone_size                 = (200, 200)
@@ -14,6 +16,10 @@ possible_point_zone             = tuple(np.asarray(image_scene_size) - np.array(
 
 position_file_pattern           = 'pos'
 click_line_pattern              = 'souris'
+zone_coodinates                 = [0, 200, 400, 600, 800]
+
+min_x_coordinate                = 100
+min_y_coordinate                = 100
 
 ## normalization_choices           = ['svd', 'svdn', 'svdne']
 

display/display_scenes_info.py

@@ -0,0 +1,54 @@
+# main imports
+import os, sys
+import argparse
+import pickle
+
+# image processing imports
+import matplotlib.pyplot as plt
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
+# define all scenes values
+scenes_list             = cfg.scenes_names
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Compute and display scenes information")
+
+    parser.add_argument('--data', type=str, help="object filename saved using pickle", required=True)
+    parser.add_argument('--scene', type=str, help="scene name to display click information", required=True, choices=cfg.scenes_names)
+
+    args = parser.parse_args()
+
+    p_data   = args.data
+    p_scene  = args.scene
+
+
+    # load data extracted by zones
+    fileObject = open(p_data, 'rb')  
+    scenes_data = pickle.load(fileObject) 
+
+    data = scenes_data[p_scene]
+    
+    plt.title(p_scene, 'with data :', p_data)
+
+    for x_i, x in enumerate(cfg.zone_coodinates):
+        plt.plot([x_i * 200, x_i * 200], [0, 800], color='red')
+
+    
+    for y_i, y in enumerate(cfg.zone_coodinates):
+        plt.plot([0, 800], [y_i * 200, y_i * 200], color='red')
+
+    plt.scatter(data['x'], data['y'])
+    plt.show()
+
+    
+
+
+if __name__== "__main__":
+    main()

display/display_zones_info.py

@@ -2,12 +2,10 @@
 import os, sys
 import argparse
 import pickle
+import numpy as np
 
-# image processing imports
-from PIL import Image
-
-from ipfml.processing import transform, segmentation
-from ipfml import utils
+# processing imports
+import matplotlib.pyplot as plt
 
 # modules imports
 sys.path.insert(0, '') # trick to enable import of main folder module
@@ -15,67 +13,44 @@ sys.path.insert(0, '') # trick to enable import of main folder module
 import custom_config as cfg
 from modules.utils import data as dt
 
-# getting configuration information
-zone_folder             = cfg.zone_folder
-min_max_filename        = cfg.min_max_filename_extension
-
-# define all scenes values
-scenes_list             = cfg.scenes_names
-scenes_indexes          = cfg.scenes_indices
-path                    = cfg.dataset_path
-zones                   = cfg.zones_indices
-seuil_expe_filename     = cfg.seuil_expe_filename
-
-output_data_folder      = cfg.output_data_folder
-
-
-image_scene_size        = cfg.image_scene_size
-image_zone_size         = cfg.image_zone_size
-possible_point_zone     = cfg.possible_point_zone
-
 
 def main():
 
     parser = argparse.ArgumentParser(description="Compute and prepare data augmentation of scenes")
 
     parser.add_argument('--data', type=str, help="object filename saved using pickle", required=True)
+    parser.add_argument('--scene', type=str, help="scene name to display click information", required=True, choices=cfg.scenes_names)
 
     args = parser.parse_args()
-
+    
     p_data   = args.data
+    p_scene  = args.scene
 
     # load data extracted by zones
     fileObject = open(p_data, 'rb')  
     scenes_data = pickle.load(fileObject) 
 
-    # get scenes list
-    scenes = os.listdir(path)
-
-    # remove min max file from scenes folder
-    scenes = [s for s in scenes if min_max_filename not in s]
-
-        # go ahead each scenes
-    for folder_scene in scenes:
+    scene_data = scenes_data[p_scene]
+    
+    # set title and zone axis
+    plt.title(p_scene, 'with data :', p_data)
 
-        scene_path = os.path.join(path, folder_scene)
+    for x_i, x in enumerate(cfg.zone_coodinates):
+        plt.plot([x_i * 200, x_i * 200], [0, 800], color='red')
 
-        # construct each zones folder name
-        zones_folder = []
-        zones_threshold = []
+    for y_i, y in enumerate(cfg.zone_coodinates):
+        plt.plot([0, 800], [y_i * 200, y_i * 200], color='red')
 
-        # get zones list info
-        for index in zones:
-            index_str = str(index)
-            if len(index_str) < 2:
-                index_str = "0" + index_str
+    x_points = []
+    y_points = []
 
-            current_zone = "zone"+index_str
-            zones_folder.append(current_zone)
+    for index, zone in scene_data.items():
 
-            zone_path = os.path.join(scene_path, current_zone)
+        x_points = np.append(x_points, zone['x'])
+        y_points = np.append(y_points, zone['y'])   
 
-            with open(os.path.join(zone_path, seuil_expe_filename)) as f:
-                zones_threshold.append(int(f.readline()))
+    plt.scatter(x_points, y_points)
+    plt.show()
 
 
 if __name__== "__main__":

generate/extract_expe_info_scenes.py

@@ -2,6 +2,7 @@
 import sys, os, argparse
 import math
 import numpy as np
+import pickle
 
 # processing imports
 import matplotlib.pyplot as plt
@@ -11,45 +12,15 @@ import scipy.stats as stats
 sys.path.insert(0, '') # trick to enable import of main folder module
 
 import custom_config as cfg
+import utils as utils_functions
 
 # variables
 data_expe_folder          = cfg.data_expe_folder
 position_file_pattern     = cfg.position_file_pattern
 click_line_pattern        = cfg.click_line_pattern
 
-# utils variables
-zone_width, zone_height   = cfg.image_zone_size
-scene_width, scene_height = cfg.image_scene_size
-nb_x_parts                = math.floor(scene_width / zone_width)
-
-min_x = 100
-min_y = 100
-
-def get_zone_index(p_x, p_y):
-
-    zone_index = math.floor(p_x / zone_width) + math.floor(p_y / zone_height) * nb_x_parts
-
-    return zone_index
-
-
-def check_coordinates(p_x, p_y):
-
-    if p_x < min_x or p_y < min_y:
-        return False
-        
-    if p_x > min_x + scene_width or p_y > min_y + scene_height:
-        return False
-    
-    return True
-
-
-def extract_click_coordinate(line):
-
-    data = line.split(' : ')[1].split(',')
-
-    p_x, p_y = (int(data[0]), int(data[1]))
-
-    return (p_x, p_y)
+min_x                     = cfg.min_x_coordinate
+min_y                     = cfg.min_y_coordinate
 
 
 def main():
@@ -105,13 +76,10 @@ def main():
                 
                 if click_line_pattern in line and scene_name in cfg.scenes_names:
                     
-                    x, y = extract_click_coordinate(line)
-
-                    points_x.append(x)
-                    points_y.append(y)
+                    x, y = utils_functions.extract_click_coordinate(line)
 
                     # only accept valid coordinates
-                    if check_coordinates(x, y):
+                    if utils_functions.check_coordinates(x, y):
                         
                         if counter < p_n:
                             scenes[scene_name]['x'].append(x - min_x)
@@ -136,33 +104,22 @@ def main():
                     if scene_name in cfg.scenes_names:
                         number_of_scenes += 1
 
-                    points_x = []
-                    points_y = []
-
                 else:
                     new_scene = False
 
                 if new_scene:
                     counter = 0
+    
+    filepath = os.path.join(cfg.extracted_data_folder, p_output)
 
+    if not os.path.exists(cfg.extracted_data_folder):
+        os.makedirs(cfg.extracted_data_folder)
 
-        #print('clicks for', subject, ':', zones_clicks)
-
-    print(scenes)
-
-    for k, v in scenes.items():
-
-        print(len(v['x']))
-        #plt.title(k)
-        #plt.scatter(v['x'], v['y'])
-        #plt.show()
+    # save information about scenes
+    with open(filepath, 'wb') as f:
+        pickle.dump(scenes, f)
 
-    '''points_x.sort()
-    hmean = np.mean(points_x)
-    hstd = np.std(points_x)
-    pdf = stats.norm.pdf(points_x, hmean, hstd)
-    plt.plot(points_x, pdf) 
-    plt.show()'''
+    print('Data object are saved into', filepath)
 
 if __name__== "__main__":
     main()

generate/extract_expe_info_zones_scenes.py

@@ -12,45 +12,15 @@ import scipy.stats as stats
 sys.path.insert(0, '') # trick to enable import of main folder module
 
 import custom_config as cfg
+import utils as utils_functions
 
 # variables
 data_expe_folder          = cfg.data_expe_folder
 position_file_pattern     = cfg.position_file_pattern
 click_line_pattern        = cfg.click_line_pattern
 
-# utils variables
-zone_width, zone_height   = cfg.image_zone_size
-scene_width, scene_height = cfg.image_scene_size
-nb_x_parts                = math.floor(scene_width / zone_width)
-
-min_x = 100
-min_y = 100
-
-def get_zone_index(p_x, p_y):
-
-    zone_index = math.floor(p_x / zone_width) + math.floor(p_y / zone_height) * nb_x_parts
-
-    return zone_index
-
-
-def check_coordinates(p_x, p_y):
-
-    if p_x < min_x or p_y < min_y:
-        return False
-        
-    if p_x >= min_x + scene_width or p_y >= min_y + scene_height:
-        return False
-    
-    return True
-
-
-def extract_click_coordinate(line):
-
-    data = line.split(' : ')[1].split(',')
-
-    p_x, p_y = (int(data[0]), int(data[1]))
-
-    return (p_x, p_y)
+min_x                     = cfg.min_x_coordinate
+min_y                     = cfg.min_y_coordinate
 
 
 def main():
@@ -58,10 +28,12 @@ def main():
     parser = argparse.ArgumentParser(description="Compute expe data into output file")
 
     parser.add_argument('--output', type=str, help="output folder expected", required=True)
+    parser.add_argument('--n', type=int, help="number of first clicks per zone wished per user")
 
     args = parser.parse_args()
 
     p_output   = args.output
+    p_n        = args.n
 
     # list all folders
     subjects = os.listdir(data_expe_folder)
@@ -98,28 +70,38 @@ def main():
         path_scene          = ""
         new_scene           = True
         number_of_scenes    = 0
-        counter             = 0
         scene_name          = ""
 
         # open pos file and extract click information 
         with open(pos_filepath, 'r') as f:
 
+            # for each subject check `p_n` on each zone
+            zones_filled = {}
+
+            # first init
+            for zone_index in cfg.zones_indices:
+                zones_filled[zone_index] = 0
+
             for line in f.readlines():
                 
                 if click_line_pattern in line and scene_name in cfg.scenes_names:
                     
-                    x, y = extract_click_coordinate(line)
+                    x, y = utils_functions.extract_click_coordinate(line)
 
                     # only accept valid coordinates
-                    if check_coordinates(x, y):
+                    if utils_functions.check_coordinates(x, y):
                         
                         p_x = x - min_x
                         p_y = y - min_y
 
-                        zone_index = get_zone_index(p_x, p_y)
+                        zone_index = utils_functions.get_zone_index(p_x, p_y)
 
-                        scenes[scene_name][zone_index]['x'].append(p_x)
-                        scenes[scene_name][zone_index]['y'].append(p_y)
+                        # check number of points saved for this specific zone
+                        # add only if wished
+                        if zones_filled[zone_index] < p_n:
+                            scenes[scene_name][zone_index]['x'].append(p_x)
+                            scenes[scene_name][zone_index]['y'].append(p_y)
+                            zones_filled[zone_index] += 1
                 
                 elif click_line_pattern not in line:
                     path_scene = line
@@ -133,14 +115,22 @@ def main():
                     if scene_name in cfg.scenes_names:
                         number_of_scenes += 1
 
+                    # reinit for each scene
+                    for zone_index in cfg.zones_indices:
+                        zones_filled[zone_index] = 0
+
                 else:
                     new_scene = False
 
-                if new_scene:
-                    counter = 0
+    filepath = os.path.join(cfg.extracted_data_folder, p_output)
 
-    with open(p_output, 'wb') as f:
+    if not os.path.exists(cfg.extracted_data_folder):
+        os.makedirs(cfg.extracted_data_folder)
+
+    with open(filepath, 'wb') as f:
         pickle.dump(scenes, f)
 
+    print('Data object are saved into', filepath)
+
 if __name__== "__main__":
     main()

generate/generate_data_augmentation_zone.py

@@ -0,0 +1,87 @@
+# main imports
+import os, sys
+import argparse
+import pickle
+
+# image processing imports
+from PIL import Image
+
+from ipfml.processing import transform, segmentation
+from ipfml import utils
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+from modules.utils import data as dt
+
+# getting configuration information
+zone_folder             = cfg.zone_folder
+min_max_filename        = cfg.min_max_filename_extension
+
+# define all scenes values
+scenes_list             = cfg.scenes_names
+scenes_indexes          = cfg.scenes_indices
+path                    = cfg.dataset_path
+zones                   = cfg.zones_indices
+seuil_expe_filename     = cfg.seuil_expe_filename
+
+output_data_folder      = cfg.output_data_folder
+
+
+image_scene_size        = cfg.image_scene_size
+image_zone_size         = cfg.image_zone_size
+possible_point_zone     = cfg.possible_point_zone
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Compute and prepare data augmentation of scenes")
+
+    parser.add_argument('--data', type=str, help="object filename saved using pickle", required=True)
+    parser.add_argument('--scene', type=str, help="scene name to display click information", required=True, choices=cfg.scenes_names)
+    parser.add_argument('--n', type=int, help="number of clics per zone wished")
+
+    args = parser.parse_args()
+    
+    p_data   = args.data
+    p_scene  = args.scene
+    p_n      = args.n
+
+    # load data extracted by zones
+    fileObject = open(p_data, 'rb')  
+    scenes_data = pickle.load(fileObject) 
+
+    scene_data = scenes_data[p_scene]
+    # get scenes list
+    scenes = os.listdir(path)
+
+    # remove min max file from scenes folder
+    scenes = [s for s in scenes if min_max_filename not in s]
+
+        # go ahead each scenes
+    for folder_scene in scenes:
+
+        scene_path = os.path.join(path, folder_scene)
+
+        # construct each zones folder name
+        zones_folder = []
+        zones_threshold = []
+
+        # get zones list info
+        for index in zones:
+            index_str = str(index)
+            if len(index_str) < 2:
+                index_str = "0" + index_str
+
+            current_zone = "zone"+index_str
+            zones_folder.append(current_zone)
+
+            zone_path = os.path.join(scene_path, current_zone)
+
+            with open(os.path.join(zone_path, seuil_expe_filename)) as f:
+                zones_threshold.append(int(f.readline()))
+
+
+if __name__== "__main__":
+    main()

utils.py

@@ -0,0 +1,41 @@
+# main imports
+import sys, math
+
+# modules imports
+sys.path.insert(0, '') # trick to enable import of main folder module
+
+import custom_config as cfg
+
+min_x = 100
+min_y = 100
+
+# utils variables
+zone_width, zone_height   = cfg.image_zone_size
+scene_width, scene_height = cfg.image_scene_size
+nb_x_parts                = math.floor(scene_width / zone_width)
+
+def get_zone_index(p_x, p_y):
+
+    zone_index = math.floor(p_x / zone_width) + math.floor(p_y / zone_height) * nb_x_parts
+
+    return zone_index
+
+
+def check_coordinates(p_x, p_y):
+
+    if p_x < min_x or p_y < min_y:
+        return False
+        
+    if p_x >= min_x + scene_width or p_y >= min_y + scene_height:
+        return False
+    
+    return True
+
+
+def extract_click_coordinate(line):
+
+    data = line.split(' : ')[1].split(',')
+
+    p_x, p_y = (int(data[0]), int(data[1]))
+
+    return (p_x, p_y)