Merge branch 'release/v0.2.2'

.gitignore

@@ -21,3 +21,6 @@ saved_models/*.h5
 *.png
 !saved_models/*.png
 .vscode
+
+# simulate models .csv file
+simulate_models*.csv

analysis/mscn_and_lab_analysis.ipynb

@@ -0,0 +1,185 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ipfml import processing, utils, metrics\n",
+    "from PIL import Image\n",
+    "from scipy import signal\n",
+    "from skimage import color\n",
+    "import scipy.stats as stats\n",
+    "import seaborn as sns\n",
+    "import cv2\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_folder = \"../fichiersSVD_light\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# MSCN analysis on Synthesis Images "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Utils functions definition"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compute_images_path(scene, prefix, indices):\n",
+    "    images_path = []\n",
+    "    for index in indices:\n",
+    "        path = os.path.join(data_folder, os.path.join(scene, prefix + index + \".png\"))\n",
+    "        print(path)\n",
+    "        images_path.append(path)\n",
+    "    return images_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_L_canal(img):\n",
+    "    img_lab = metrics.get_LAB_L(img)\n",
+    "    img_lab = np.asarray(img_lab, 'uint8')\n",
+    "    \n",
+    "    return Image.fromarray(img_lab)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_MSCN_canal(img):\n",
+    "    img_mscn = processing.get_mscn_coefficients(img)\n",
+    "    img_mscn = np.asarray(utils.normalize_2D_arr(img_mscn)*255, 'uint8')\n",
+    "    \n",
+    "    return Image.fromarray(img_mscn)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Scenes MSCN variance analysis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Cuisine01 scene "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scene_name = \"Cuisine01\"\n",
+    "prefix_name = \"cuisine01_\"\n",
+    "image_indices = [\"00050\", \"00100\", \"00200\", \"00300\", \"00500\", \"00900\",\"01200\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "../fichiersSVD_light/Cuisine01/cuisine01_00050.png\n",
+      "../fichiersSVD_light/Cuisine01/cuisine01_00100.png\n",
+      "../fichiersSVD_light/Cuisine01/cuisine01_00200.png\n",
+      "../fichiersSVD_light/Cuisine01/cuisine01_00300.png\n",
+      "../fichiersSVD_light/Cuisine01/cuisine01_00500.png\n",
+      "../fichiersSVD_light/Cuisine01/cuisine01_00900.png\n",
+      "../fichiersSVD_light/Cuisine01/cuisine01_01200.png\n"
+     ]
+    }
+   ],
+   "source": [
+    "images_path = compute_images_path(scene_name, prefix_name, image_indices)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "get_L_canal(processing.divide_in_blocks(Image.open(images_path[0]), (200, 200))[10]).save('tmp_images/cuisine01_zone10_00050_lab.png')\n",
+    "get_L_canal(processing.divide_in_blocks(Image.open(images_path[5]), (200, 200))[10]).save('tmp_images/cuisine01_zone10_01200_lab.png')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "get_MSCN_canal(processing.divide_in_blocks(Image.open(images_path[0]), (200, 200))[10]).save('tmp_images/cuisine01_zone10_00050_mscn.png')\n",
+    "get_MSCN_canal(processing.divide_in_blocks(Image.open(images_path[5]), (200, 200))[10]).save('tmp_images/cuisine01_zone10_01200_mscn.png')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "processing.divide_in_blocks(Image.open(images_path[0]), (200, 200))[10].save('tmp_images/cuisine01_zone10_noisy.png')\n",
+    "processing.divide_in_blocks(Image.open(images_path[5]), (200, 200))[10].save('tmp_images/cuisine01_zone10_ref.png')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "thesis-venv",
+   "language": "python",
+   "name": "thesis-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

analysis/statistic_analysis_svd.ipynb

@@ -0,0 +1,270 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ipfml import processing\n",
+    "from ipfml import utils\n",
+    "from ipfml import metrics\n",
+    "from PIL import Image\n",
+    "from scipy import signal\n",
+    "from skimage import color\n",
+    "import scipy.stats as stats\n",
+    "import seaborn as sns\n",
+    "import cv2\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import os\n",
+    "\n",
+    "from pylab import *\n",
+    "from skimage import data, io, color\n",
+    "from skimage import feature\n",
+    "\n",
+    "import cv2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_folder = \"../fichiersSVD_light\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Local Binary Pattern analysis on Synthesis Images for noise dectection "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Utils functions definition"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compute_images_path(dict_data):\n",
+    "    \n",
+    "    all_images_path = []\n",
+    "    for cur_dict in dict_data:\n",
+    "        scene = cur_dict['name']\n",
+    "        prefix = cur_dict['prefix']\n",
+    "        indices = cur_dict['indices']\n",
+    "\n",
+    "        scene_images_path = []\n",
+    "        for index in indices:\n",
+    "            path = os.path.join(data_folder, os.path.join(scene, prefix + index + \".png\"))\n",
+    "            scene_images_path.append(path)\n",
+    "            \n",
+    "        all_images_path.append(scene_images_path)\n",
+    "            \n",
+    "    return all_images_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def display_sv_data(dict_data, interval, all_images_path):\n",
+    "    \n",
+    "    sv_values = []\n",
+    "    plt.figure(figsize=(25, 20))\n",
+    "    begin, end = interval\n",
+    "    \n",
+    "    for id_dict, cur_dict in enumerate(dict_data):\n",
+    "        \n",
+    "        scene_name = cur_dict['name']\n",
+    "        image_indices = cur_dict['indices']\n",
+    "        scene_sv_values = []\n",
+    "    \n",
+    "        for id_img, img_path in enumerate(all_images_path[id_dict]):\n",
+    "            img = Image.open(img_path)\n",
+    "            print(img_path)\n",
+    "            \n",
+    "            blocks = processing.divide_in_blocks(img, (200, 200))\n",
+    "            block = np.array(blocks[0])\n",
+    "            \n",
+    "            if block.ndim == 3:\n",
+    "                U, s, V = processing.get_LAB_L_SVD(block)\n",
+    "            else:\n",
+    "                U, s, V = metrics.get_SVD(block)\n",
+    "                \n",
+    "            data = s[begin:end]\n",
+    "            plt.plot(data, label=scene_name + '_' + str(image_indices[id_img]))\n",
+    "            scene_sv_values.append(data)\n",
+    "            \n",
+    "        sv_values.append(scene_sv_values)\n",
+    "\n",
+    "    plt.legend(fontsize=18)\n",
+    "    plt.show()\n",
+    "    return sv_values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_sv_data(cur_dict, interval, images_path_scene, norm=False):\n",
+    "    \n",
+    "    scene_name = cur_dict['name']\n",
+    "    image_indices = cur_dict['indices']\n",
+    "    zone = cur_dict['zone']\n",
+    "    scene_sv_values = []\n",
+    "    begin, end = interval\n",
+    "    \n",
+    "    plt.figure(figsize=(25, 15))\n",
+    "        \n",
+    "    for id_img, img_path in enumerate(images_path_scene):\n",
+    "        img = Image.open(img_path)\n",
+    "        \n",
+    "        img = processing.divide_in_blocks(img, (200, 200))[zone]\n",
+    "        \n",
+    "        # Convert to L canal\n",
+    "        img_grey = np.asarray(metrics.get_LAB_L(img), 'uint8')\n",
+    "        \n",
+    "        data_new = whiten(img_grey)\n",
+    "        data_new = np.diagonal(data_new)\n",
+    "        \n",
+    "        data = processing.get_LAB_L_SVD_s(img)\n",
+    "        \n",
+    "        #Image.fromarray(new_img).show()\n",
+    "        \n",
+    "        if norm:\n",
+    "            data_new = utils.normalize_arr(data_new)\n",
+    "            data = utils.normalize_arr(data)\n",
+    "        \n",
+    "        plt.plot(data, label=scene_name + '_' + str(image_indices[id_img] + ' (new)'))\n",
+    "        plt.plot(data_new, label=scene_name + '_' + str(image_indices[id_img]))\n",
+    "        scene_sv_values.append(data)\n",
+    "    \n",
+    "    plt.legend(fontsize=18)\n",
+    "    plt.show()\n",
+    "    return scene_sv_values\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class LocalBinaryPatterns:\n",
+    "\tdef __init__(self, numPoints, radius):\n",
+    "\t\t# store the number of points and radius\n",
+    "\t\tself.numPoints = numPoints\n",
+    "\t\tself.radius = radius\n",
+    " \n",
+    "\tdef describe(self, image, eps=1e-7):\n",
+    "\t\t# compute the Local Binary Pattern representation\n",
+    "\t\t# of the image, and then use the LBP representation\n",
+    "\t\t# to build the histogram of patterns\n",
+    "\t\tlbp = feature.local_binary_pattern(image, self.numPoints,\n",
+    "\t\t\tself.radius, method=\"uniform\")\n",
+    "\t\t(hist, _) = np.histogram(lbp.ravel(),\n",
+    "\t\t\tbins=np.arange(0, self.numPoints + 3),\n",
+    "\t\t\trange=(0, self.numPoints + 2))\n",
+    " \n",
+    "\t\t# normalize the histogram\n",
+    "\t\thist = hist.astype(\"float\")\n",
+    "\t\thist /= (hist.sum() + eps)\n",
+    " \n",
+    "\t\t# return the histogram of Local Binary Patterns\n",
+    "\t\treturn hist"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Scenes zones data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# start 00020 - ref 00900 - step 10\n",
+    "dict_appart = {'name': 'Appart1opt02', \n",
+    "               'prefix': 'appartAopt_', \n",
+    "               'indices': [\"00020\", \"00200\", \"00300\", \"00900\"],\n",
+    "               'zone': 6}\n",
+    "\n",
+    "# start 00050 - ref 01200 - step 10\n",
+    "dict_cuisine = {'name': 'Cuisine01', \n",
+    "               'prefix': 'cuisine01_', \n",
+    "               'indices': [\"00050\", \"00400\", \"01200\"],\n",
+    "               'zone': 6}\n",
+    "\n",
+    "# start 00020 - ref 00950 - step 10\n",
+    "dict_sdb_c = {'name': 'SdbCentre', \n",
+    "               'prefix': 'SdB2_', \n",
+    "               'indices': [\"00020\", \"00400\", \"00950\"],\n",
+    "               'zone': 6}\n",
+    "\n",
+    "# start 00020 - ref 00950 - step 10\n",
+    "dict_sdb_d = {'name': 'SdbDroite', \n",
+    "               'prefix': 'SdB2_D_', \n",
+    "               'indices': [\"00020\", \"00400\", \"00950\"],\n",
+    "               'zone': 6}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_dicts = [dict_appart, dict_cuisine, dict_sdb_c, dict_sdb_d]\n",
+    "interval = (30, 200)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "images_path = compute_images_path(all_dicts)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "thesis-venv",
+   "language": "python",
+   "name": "thesis-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

analysis/svd_filters_analysis.ipynb

@@ -0,0 +1,367 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ipfml import processing\n",
+    "from ipfml import utils\n",
+    "from ipfml import metrics\n",
+    "from PIL import Image\n",
+    "from scipy import signal\n",
+    "from skimage import color\n",
+    "import scipy.stats as stats\n",
+    "import seaborn as sns\n",
+    "import cv2\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import os\n",
+    "import math\n",
+    "\n",
+    "from scipy.signal import medfilt2d, wiener, cwt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_folder = \"../fichiersSVD_light\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# SVD filter analysis on zones of Synthesis Images "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Utils functions definition"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compute_images_path(dict_data):\n",
+    "    scene = dict_data['name']\n",
+    "    prefix = dict_data['prefix']\n",
+    "    indices = dict_data['indices']\n",
+    "    \n",
+    "    images_path = []\n",
+    "    for index in indices:\n",
+    "        path = os.path.join(data_folder, os.path.join(scene, prefix + index + \".png\"))\n",
+    "        print(path)\n",
+    "        images_path.append(path)\n",
+    "    return images_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_images_zones(dict_data, images_path):\n",
+    "    \n",
+    "    zones_indices = dict_data['zones']\n",
+    "    zones_img = []\n",
+    "    \n",
+    "    for path in images_path:\n",
+    "        img = Image.open(path)\n",
+    "        zones = processing.divide_in_blocks(img, (200, 200))\n",
+    "        \n",
+    "        zones_list = []\n",
+    "        \n",
+    "        for id_zone in zones_indices:\n",
+    "            zones_list.append(zones[id_zone])\n",
+    "            \n",
+    "        zones_img.append(zones_list)\n",
+    "        \n",
+    "    return zones_img"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def display_sv_data(dict_data, zones_data, interval, _norm=False):\n",
+    "    \n",
+    "    scene_name = dict_data['name']\n",
+    "    image_indices = dict_data['indices']\n",
+    "    zones_indices = dict_data['zones']\n",
+    "    colors = ['C0', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9']\n",
+    "    \n",
+    "    plt.figure(figsize=(25, 20))\n",
+    "    \n",
+    "    sv_data = []\n",
+    "    begin, end = interval\n",
+    "    for id_img, zones in enumerate(zones_data):\n",
+    "        \n",
+    "        for id_zone, zone in enumerate(zones):\n",
+    "            U, s, V = processing.get_LAB_L_SVD(zone)\n",
+    "        \n",
+    "            data = s[begin:end]\n",
+    "            \n",
+    "            if _norm:\n",
+    "                data = utils.normalize_arr(data)\n",
+    "                \n",
+    "            plt.plot(data, \n",
+    "                     color=colors[id_zone], \n",
+    "                     label='Zone ' + str(zones_indices[id_zone]) + ' of ' + scene_name + '_' + str(image_indices[id_img]))\n",
+    "            \n",
+    "    plt.legend(fontsize=18)\n",
+    "    plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Useful function\n",
+    "\n",
+    "def get_highest_values(arr, n):\n",
+    "    return np.array(arr).argsort()[-n:][::-1]\n",
+    "\n",
+    "def get_lowest_values(arr, n):\n",
+    "    return np.array(arr).argsort()[::-1][-n:][::-1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_sv_std(arr):\n",
+    "    arr = np.array(arr)\n",
+    "    images = [arr]\n",
+    "    \n",
+    "    # Apply list of filter on arr\n",
+    "    images.append(medfilt2d(arr, [3, 3]))\n",
+    "    images.append(medfilt2d(arr, [5, 5]))\n",
+    "    images.append(wiener(arr, [3, 3]))\n",
+    "    images.append(wiener(arr, [5, 5]))\n",
+    "    \n",
+    "    sv_vector = []\n",
+    "    # for each new image apply SVD and get SV \n",
+    "    for img in images:\n",
+    "        s = metrics.get_SVD_s(img)\n",
+    "        sv_vector.append(s)\n",
+    "        \n",
+    "    sv_array = np.array(sv_vector)\n",
+    "    \n",
+    "    _, len = sv_array.shape\n",
+    "    \n",
+    "    sv_std = []\n",
+    "    \n",
+    "    # normalize each SV vectors and compute standard deviation for each sub vectors\n",
+    "    for i in range(len):\n",
+    "        sv_array[:, i] = utils.normalize_arr(sv_array[:, i])\n",
+    "        sv_std.append(np.std(sv_array[:, i]))\n",
+    "        \n",
+    "    \n",
+    "    return np.array(sv_std)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Scenes zones data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# start 00020 - ref 00900 - step 10\n",
+    "dict_appart = {'name': 'Appart1opt02', \n",
+    "               'prefix': 'appartAopt_', \n",
+    "               'indices': [\"00020\", \"00200\", \"00900\"],\n",
+    "               'zones': [3, 6]}\n",
+    "\n",
+    "# start 00050 - ref 01200 - step 10\n",
+    "dict_cuisine = {'name': 'Cuisine01', \n",
+    "               'prefix': 'cuisine01_', \n",
+    "               'indices': [\"00050\", \"00400\", \"01200\"],\n",
+    "               'zones': [3, 6]}\n",
+    "\n",
+    "# start 00020 - ref 00950 - step 10\n",
+    "dict_sdb_c = {'name': 'SdbCentre', \n",
+    "               'prefix': 'SdB2_', \n",
+    "               'indices': [\"00020\", \"00400\", \"00950\"],\n",
+    "               'zones': [3, 6]}\n",
+    "\n",
+    "# start 00020 - ref 00950 - step 10\n",
+    "dict_sdb_d = {'name': 'SdbDroite', \n",
+    "               'prefix': 'SdB2_D_', \n",
+    "               'indices': [\"00020\", \"00400\", \"00950\"],\n",
+    "               'zones': [2, 3, 10, 13]}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "current_dict = dict_appart\n",
+    "interval = (30, 200)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "../fichiersSVD_light/Appart1opt02/appartAopt_00020.png\n",
+      "../fichiersSVD_light/Appart1opt02/appartAopt_00200.png\n",
+      "../fichiersSVD_light/Appart1opt02/appartAopt_00900.png\n"
+     ]
+    }
+   ],
+   "source": [
+    "images_path = compute_images_path(current_dict)\n",
+    "zones_data = get_images_zones(current_dict, images_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/jbuisine/.pyenv/versions/thesis-venv/lib/python3.6/site-packages/scipy/signal/signaltools.py:966: RuntimeWarning: divide by zero encountered in true_divide\n",
+      "  res *= (1 - noise / lVar)\n",
+      "/home/jbuisine/.pyenv/versions/thesis-venv/lib/python3.6/site-packages/scipy/signal/signaltools.py:966: RuntimeWarning: invalid value encountered in multiply\n",
+      "  res *= (1 - noise / lVar)\n"
+     ]
+    }
+   ],
+   "source": [
+    "lab_image = metrics.get_LAB_L(zones_data[0][0])\n",
+    "\n",
+    "sv_std_vector = get_sv_std(lab_image)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([4.56454151e-12, 5.98443803e-01, 2.92380551e-01, 3.11862186e+00,\n",
+       "       1.83881976e+00, 1.32616590e+00, 2.31162957e+00, 3.26004542e+00,\n",
+       "       3.69558983e+00, 4.40479644e+00, 5.86361202e+00, 4.57210378e+00,\n",
+       "       4.01233699e+00, 9.91121754e+00, 1.49335463e+01, 1.02598281e+01,\n",
+       "       9.33149634e+00, 9.02596321e+00, 7.37994999e+00, 6.44933954e+00,\n",
+       "       1.41221182e+01, 6.92716771e+00, 6.66692086e+00, 1.96695779e+02,\n",
+       "       1.35313092e+01, 4.82088913e+00, 1.30004193e+01, 1.26877063e+01,\n",
+       "       1.19550988e+01, 1.54599579e+01, 1.12188124e+01, 8.25570286e+00,\n",
+       "       8.47955315e+00, 1.20754110e+01, 7.76856416e+00, 1.55686219e+01,\n",
+       "       1.13219704e+01, 4.78254950e+01, 1.61024565e+01, 1.05388706e+01,\n",
+       "       5.00185479e+01, 1.50232416e+01, 4.66821321e+01, 5.16188334e+01,\n",
+       "       4.94272850e+00, 5.06946699e+01, 4.60741401e+01, 4.95388007e+01,\n",
+       "       1.62546781e+01, 1.70462990e+01, 1.66268801e+01, 4.64068807e+01,\n",
+       "       4.87707118e+01, 5.18047108e+01, 2.42465168e+01, 1.75765638e+01,\n",
+       "       4.80224924e+01, 4.53850435e+01, 2.49940296e+01, 2.32956634e+01,\n",
+       "       2.44845973e+01, 1.98151482e+01, 2.28386331e+01, 1.95685413e+01,\n",
+       "       2.33535639e+01, 5.43324597e+01, 4.83919154e+01, 4.44866045e+01,\n",
+       "       2.46294407e+01, 2.53035119e+01, 1.90294314e+01, 4.86445357e+01,\n",
+       "       4.51782154e+01, 5.40772949e+01, 1.81583448e+01, 4.41443592e+01,\n",
+       "       3.69757725e+01, 5.23320448e+01, 4.21467616e+01, 5.33122822e+01,\n",
+       "       4.24121613e+01, 2.70014670e+01, 4.20051035e+01, 2.21602652e+01,\n",
+       "       2.59550074e+01, 2.86226215e+01, 2.06325539e+01, 2.61912490e+01,\n",
+       "       3.34594682e+01, 3.02981684e+01, 3.05140681e+01, 3.15200613e+01,\n",
+       "       3.56035468e+01, 2.00897239e+01, 3.60414295e+01, 2.95487956e+01,\n",
+       "       1.78073124e+01, 2.08486817e+01, 3.28774249e+01, 3.17002386e+01])"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "indices = get_highest_values(sv_std_vector, 100)\n",
+    "\n",
+    "s_lab_image = metrics.get_SVD_s(lab_image)\n",
+    "\n",
+    "s_lab_image[indices]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([  1,   3,   0,  26,  28,  29,  23,  30,  27,  31,  22,  21,  20,\n",
+       "        24,  25,  11,  14,  18,   6,  15,  34,  17,  32,  37,  35,   2,\n",
+       "        12,  33,  19,  36,  38,  39,  41,  40,   7,  47,  44,  42,  43,\n",
+       "        45,  49,  46,  13,  48,   9,  50,  16,  10,   8,  53,  54,  55,\n",
+       "        51,  56,  52,  61,  57,  58,  59,  60,  63,  62,  66,  65,  64,\n",
+       "        67,  69, 197,  70, 104, 103, 105,  68,  72,  73,  71, 106,   4,\n",
+       "       107, 108, 117, 109,  74, 111,  78, 145, 114, 123, 115, 102,  97,\n",
+       "       152, 144, 101, 130, 131, 129, 127, 120, 116])"
+      ]
+     },
+     "execution_count": 52,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "get_lowest_values(sv_std_vector, 100)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "thesis-venv",
+   "language": "python",
+   "name": "thesis-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

analysis/svd_ica_scenes_analysis.ipynb

@@ -280,7 +280,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Essayer de garder uniquement les composante faibles et comparer avec ICA"
+    "Essayer de garder uniquement les composantes faibles et comparer avec ICA"
    ]
   },
   {

analysis/svd_ipca_scenes_analysis.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -26,7 +26,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -49,7 +49,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -73,7 +73,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -121,7 +121,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -148,7 +148,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -158,7 +158,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -206,28 +206,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "../fichiersSVD_light/Appart1opt02/appartAopt_00020.png\n",
-      "../fichiersSVD_light/Appart1opt02/appartAopt_00200.png\n",
-      "../fichiersSVD_light/Appart1opt02/appartAopt_00900.png\n",
-      "../fichiersSVD_light/Cuisine01/cuisine01_00050.png\n",
-      "../fichiersSVD_light/Cuisine01/cuisine01_00400.png\n",
-      "../fichiersSVD_light/Cuisine01/cuisine01_01200.png\n",
-      "../fichiersSVD_light/SdbCentre/SdB2_00020.png\n",
-      "../fichiersSVD_light/SdbCentre/SdB2_00400.png\n",
-      "../fichiersSVD_light/SdbCentre/SdB2_00950.png\n",
-      "../fichiersSVD_light/SdbDroite/SdB2_D_00020.png\n",
-      "../fichiersSVD_light/SdbDroite/SdB2_D_00400.png\n",
-      "../fichiersSVD_light/SdbDroite/SdB2_D_00950.png\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "all_trunc_svd_sv = []\n",
     "\n",
@@ -255,7 +236,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {

display_simulation_curves.py

@@ -100,7 +100,17 @@ def main():
     args = parser.parse_args()
 
     p_folder = args.folder
-    p_model = args.model
+
+    if args.model:
+        p_model = args.model
+    else:
+        # find p_model from folder if model arg not given (folder path need to have model name)
+        if p_folder.split('/')[-1]:
+            p_model = p_folder.split('/')[-1]
+        else:
+            p_model = p_folder.split('/')[-2]
+    
+    print(p_model)
 
     display_curves(p_folder, p_model)
 

generate_all_simulate_curves.sh

@@ -0,0 +1,6 @@
+for file in "threshold_map"/*; do
+
+    echo ${file}
+
+    python display_simulation_curves.py --folder ${file}
+done

generate_data_model_random.py

@@ -44,7 +44,6 @@ generic_output_file_svd = '_random.csv'
 min_value_interval      = sys.maxsize
 max_value_interval      = 0
 
-
 def construct_new_line(path_seuil, interval, line, choice, each, norm):
     begin, end = interval
 

julia_train_model.jl

@@ -0,0 +1,36 @@
+using DataFrames
+using CSV
+using PyCall, JLD, PyCallJLD
+using ArgParse
+
+
+function main(args)
+
+    # initialize the settings (the description is for the help screen)
+    s = ArgParseSettings(description = "Example 1 for argparse.jl: minimal usage.")
+
+    @add_arg_table s begin
+        "--data"    
+        #    arg_type = Int           # only Int arguments allowed
+        #    nargs = '?'              # '?' means optional argument
+        #    default = 0              # this is used when the option is not passed
+        #    constant = 1             # this is used if --opt1 is paseed with no argument
+            help = "Data file for train and test"
+        
+        "--output"
+            help = "output model name"
+
+        "choice"
+            help = "Model choice"
+    end
+
+    parsed_args = parse_args(s) # the result is a Dict{String,Any}
+
+    println(parsed_args["data"])
+    println("Parsed args:")
+    for (key,val) in parsed_args
+        println("  $key  =>  $(repr(val))")
+    end
+end
+
+main(ARGS)

modules/utils/config.py

@@ -35,7 +35,7 @@ cycle_scenes_indices            = ['E', 'I']
 normalization_choices           = ['svd', 'svdn', 'svdne']
 zones_indices                   = np.arange(16)
 
-metric_choices_labels           = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed', 'mscn_var_4', 'mscn_var_16', 'mscn_var_64', 'mscn_var_16_max', 'mscn_var_64_max', 'ica_diff', 'svd_trunc_diff', 'ipca_diff']
+metric_choices_labels           = ['lab', 'mscn', 'low_bits_2', 'low_bits_3', 'low_bits_4', 'low_bits_5', 'low_bits_6','low_bits_4_shifted_2', 'sub_blocks_stats', 'sub_blocks_area', 'sub_blocks_stats_reduced', 'sub_blocks_area_normed', 'mscn_var_4', 'mscn_var_16', 'mscn_var_64', 'mscn_var_16_max', 'mscn_var_64_max', 'ica_diff', 'svd_trunc_diff', 'ipca_diff', 'svd_reconstruct', 'highest_sv_std_filters', 'lowest_sv_std_filters']
 
 keras_epochs                    = 500
 keras_batch                     = 32

modules/utils/data.py

@@ -6,6 +6,9 @@ from skimage import color
 from sklearn.decomposition import FastICA
 from sklearn.decomposition import IncrementalPCA
 from sklearn.decomposition import TruncatedSVD
+from numpy.linalg import svd as lin_svd
+
+from scipy.signal import medfilt2d, wiener, cwt
 
 import numpy as np
 
@@ -248,8 +251,79 @@ def get_svd_data(data_type, block):
         U, s, V = metrics.get_SVD(reduced_image)
         data = s
 
+    if data_type == 'svd_reconstruct':
+
+        reconstructed_interval = (90, 200)
+        begin, end = reconstructed_interval
+
+        lab_img = metrics.get_LAB_L(block)
+        lab_img = np.array(lab_img, 'uint8')
+
+        U, s, V = lin_svd(lab_img, full_matrices=True)
+
+        smat = np.zeros((end-begin, end-begin), dtype=complex)
+        smat[:, :] = np.diag(s[begin:end])
+        output_img = np.dot(U[:, begin:end],  np.dot(smat, V[begin:end, :]))
+
+        output_img = np.array(output_img, 'uint8')
+
+        data = metrics.get_SVD_s(output_img)
+
+    if 'sv_std_filters' in data_type:
+
+        # convert into lab by default to apply filters
+        lab_img = metrics.get_LAB_L(block)
+        arr = np.array(lab_img)
+        images = []
+        
+        # Apply list of filter on arr
+        images.append(medfilt2d(arr, [3, 3]))
+        images.append(medfilt2d(arr, [5, 5]))
+        images.append(wiener(arr, [3, 3]))
+        images.append(wiener(arr, [5, 5]))
+        
+        # By default computation of current block image
+        s_arr = metrics.get_SVD_s(arr)
+        sv_vector = [s_arr]
+
+        # for each new image apply SVD and get SV 
+        for img in images:
+            s = metrics.get_SVD_s(img)
+            sv_vector.append(s)
+            
+        sv_array = np.array(sv_vector)
+        
+        _, len = sv_array.shape
+        
+        sv_std = []
+        
+        # normalize each SV vectors and compute standard deviation for each sub vectors
+        for i in range(len):
+            sv_array[:, i] = utils.normalize_arr(sv_array[:, i])
+            sv_std.append(np.std(sv_array[:, i]))
+        
+        indices = []
+
+        if 'lowest' in data_type:
+            indices = get_lowest_values(sv_std, 200)
+
+        if 'highest' in data_type:
+            indices = get_highest_values(sv_std, 200)
+
+        # data are arranged following std trend computed
+        data = s_arr[indices]
+
     return data
 
+
+def get_highest_values(arr, n):
+    return np.array(arr).argsort()[-n:][::-1]
+
+
+def get_lowest_values(arr, n):
+    return np.array(arr).argsort()[::-1][-n:][::-1]
+
+
 def _get_mscn_variance(block, sub_block_size=(50, 50)):
 
     blocks = processing.divide_in_blocks(block, sub_block_size)

runAll_maxwell_custom.sh

@@ -18,7 +18,8 @@ fi
 
 for size in {"4","8","16","26","32","40"}; do
 
-    for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff"}; do
+    # for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
+    for metric in {"highest_sv_std_filters","lowest_sv_std_filters"}; do
         bash generateAndTrain_maxwell_custom.sh ${size} ${metric}
     done
 done

runAll_maxwell_custom_center.sh

@@ -18,7 +18,8 @@ fi
 
 for size in {"4","8","16","26","32","40"}; do
 
-    for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff"}; do
+    # for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
+    for metric in {"highest_sv_std_filters","lowest_sv_std_filters"}; do
         bash generateAndTrain_maxwell_custom_center.sh ${size} ${metric}
     done
 done

runAll_maxwell_custom_split.sh

@@ -18,7 +18,8 @@ fi
 
 for size in {"4","8","16","26","32","40"}; do
 
-    for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff"}; do
+    #for metric in {"lab","mscn","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","svd_trunc_diff","ipca_diff","svd_reconstruct"}; do
+    for metric in {"highest_sv_std_filters","lowest_sv_std_filters"}; do
         bash generateAndTrain_maxwell_custom_split.sh ${size} ${metric}
     done
 done

run_maxwell_simulation_custom.sh

@@ -8,7 +8,7 @@ scenes="A, D, G, H"
 VECTOR_SIZE=200
 
 for size in {"4","8","16","26","32","40"}; do
-    for metric in {"lab","mscn","mscn_revisited","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2"}; do
+    for metric in {"lab","mscn","mscn_revisited","low_bits_2","low_bits_3","low_bits_4","low_bits_5","low_bits_6","low_bits_4_shifted_2","ica_diff","ipca_diff","svd_trunc_diff","svd_reconstruct"}; do
 
         half=$(($size/2))
         start=-$half

simulate_models.csv

@@ -1,5 +0,0 @@
-ensemble_model_N40_B0_E40_nb_zones_10_lab_svdne_corr_L0_H1
-ensemble_model_N35_B0_E35_nb_zones_12_lab_svdne_corr_L0_H1
-ensemble_model_N40_B0_E40_nb_zones_12_lab_svdne_corr_L0_H1
-svm_model_N25_B0_E25_nb_zones_4_lab_svdne_corr_L0_H1
-ensemble_model_N40_B0_E40_nb_zones_8_lab_svdne_corr_L0_H1

simulate_models_keras.csv

@@ -1,5 +0,0 @@
-deep_keras_N35_B0_E35_nb_zones_10_lab_svd_corr_L0_H1
-deep_keras_N25_B0_E25_nb_zones_10_lab_svdne_corr_L0_H1
-deep_keras_N30_B0_E30_nb_zones_6_lab_svdne_corr_L0_H1
-deep_keras_N5_B0_E5_nb_zones_12_lab_svdn_corr_L1_H1
-deep_keras_N25_B0_E25_nb_zones_6_lab_svdne_corr_L0_H1

simulate_models_keras_corr.csv

@@ -1,5 +0,0 @@
-deep_keras_N35_B0_E35_nb_zones_10_lab_svd_corr_L0_H1
-deep_keras_N25_B0_E25_nb_zones_10_lab_svdne_corr_L0_H1
-deep_keras_N30_B0_E30_nb_zones_6_lab_svdne_corr_L0_H1
-deep_keras_N5_B0_E5_nb_zones_12_lab_svdn_corr_L1_H1
-deep_keras_N25_B0_E25_nb_zones_6_lab_svdne_corr_L0_H1