Thesis-NoiseDetection-attributes/generate/generate_all_data_augmentation.py

# main imports
import sys, os, argparse
import numpy as np
import random
import time
import json

# image processing imports
from PIL import Image

from ipfml.processing import transform, segmentation
from ipfml import utils

# modules imports
sys.path.insert(0, '') # trick to enable import of main folder module

import custom_config as cfg
from modules.utils import data as dt
from data_attributes import get_image_features


# getting configuration information
zone_folder             = cfg.zone_folder
min_max_filename        = cfg.min_max_filename_extension

# define all scenes values
scenes_list             = cfg.scenes_names
scenes_indexes          = cfg.scenes_indices
choices                 = cfg.normalization_choices
zones                   = cfg.zones_indices
seuil_expe_filename     = cfg.seuil_expe_filename

features_choices        = cfg.features_choices_labels
output_data_folder      = cfg.output_data_folder

data_augmented_filename = cfg.data_augmented_filename
generic_output_file_svd = '_random.csv'

def generate_data_svd(data_type, mode, path):
    """
    @brief Method which generates all .csv files from scenes
    @param data_type,  feature choice
    @param mode, normalization choice
    @param path, data augmented path
    @return nothing
    """

    scenes = os.listdir(path)
    # remove min max file from scenes folder
    scenes = [s for s in scenes if min_max_filename and generic_output_file_svd not in s]

    # keep in memory min and max data found from data_type
    min_val_found = sys.maxsize
    max_val_found = 0

    data_min_max_filename = os.path.join(path, data_type + min_max_filename)
    data_filename = os.path.join(path, data_augmented_filename)

    # getting output filename
    output_svd_filename = data_type + "_" + mode + generic_output_file_svd

    current_file = open(os.path.join(path, output_svd_filename), 'w')

    with open(data_filename, 'r') as f:

        lines = f.readlines()
        number_of_images = len(lines)

        for index, line in enumerate(lines):        
            
            data = line.split(';')

            scene_name = data[0]
            number_of_samples = data[2]
            label_img = data[3]
            img_path = data[4].replace('\n', '')

            block = Image.open(os.path.join(path, img_path))
         
            ###########################
            # feature computation part #
            ###########################

            data = get_image_features(data_type, block)

            ##################
            # Data mode part #
            ##################

            # modify data depending mode
            if mode == 'svdne':

                # getting max and min information from min_max_filename
                with open(data_min_max_filename, 'r') as f:
                    min_val = float(f.readline())
                    max_val = float(f.readline())

                data = utils.normalize_arr_with_range(data, min_val, max_val)

            if mode == 'svdn':
                data = utils.normalize_arr(data)

            # save min and max found from dataset in order to normalize data using whole data known
            if mode == 'svd':

                current_min = data.min()
                current_max = data.max()

                if current_min < min_val_found:
                    min_val_found = current_min

                if current_max > max_val_found:
                    max_val_found = current_max

            # add of index
            current_file.write(scene_name + ';' + number_of_samples + ';' + label_img + ';')

            for val in data:
                current_file.write(str(val) + ";")

            print(data_type + "_" + mode + " - " + "{0:.2f}".format((index + 1) / number_of_images * 100.) + "%")
            sys.stdout.write("\033[F")

            current_file.write('\n')

        print('\n')

    # save current information about min file found
    if mode == 'svd':
        with open(data_min_max_filename, 'w') as f:
            f.write(str(min_val_found) + '\n')
            f.write(str(max_val_found) + '\n')

    print("%s_%s : end of data generation\n" % (data_type, mode))


def main():

    parser = argparse.ArgumentParser(description="Compute and prepare data of feature of all scenes (keep in memory min and max value found)")

    parser.add_argument('--feature', type=str, 
                                    help="feature choice in order to compute data (use 'all' if all features are needed)")
    parser.add_argument('--folder', type=str, help="folder which contains the whole dataset")

    args = parser.parse_args()

    p_feature = args.feature
    p_folder  = args.folder

    # generate all or specific feature data
    if p_feature == 'all':
        for m in features_choices:
            generate_data_svd(m, 'svd', p_folder)
            generate_data_svd(m, 'svdn', p_folder)
            generate_data_svd(m, 'svdne', p_folder)
    else:

        if p_feature not in features_choices:
            raise ValueError('Unknown feature choice : ', features_choices)
            
        generate_data_svd(p_feature, 'svd', p_folder)
        generate_data_svd(p_feature, 'svdn', p_folder)
        generate_data_svd(p_feature, 'svdne', p_folder)

if __name__== "__main__":
    main()