{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = \"../data\"\n",
    "surrogate_data_folder = os.path.join(data_folder, \"surrogate\", \"data\")\n",
    "surrogate_static_folder = os.path.join(data_folder, \"surrogate\", \"static\")\n",
    "log_folder = os.path.join(data_folder, \"logs\")\n",
    "\n",
    "output_images_folder = 'images'\n",
    "if not os.path.exists(output_images_folder):\n",
    "    os.makedirs(output_images_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What to display ?\n",
    "\n",
    "- **1. Curve of performance runs during search process (`surrogate/data`) :**\n",
    "    - best solution score (in means from 5 runs)\n",
    "    - comparisons of different orders\n",
    "    - comparisons of different LS iterations per orders\n",
    "    - extract times of computation (`data/logs`)\n",
    "    - surrogate model performance during training (r²)\n",
    "\n",
    "\n",
    "- **2. From only best last solutions found displays :**\n",
    "    - bar plot of best attribute selected for classification per order\n",
    "    - same for the two orders "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Curve of performances runs during search process (`surrogate/data`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fixed parameters\n",
    "ILS = 1000\n",
    "POP = 20\n",
    "N_RUNS = 5\n",
    "SS = 50 # start surrogate training after at least 50 real evaluations\n",
    "SE = 10 # train every 10 real evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_params(filename):\n",
    "    #pop_size = filename.split('-')[2].replace('POP_', '')\n",
    "    ls_iter = filename.split('-')[3].replace('LS_', '')\n",
    "    s_order = filename.split('-')[5].replace('SO_', '')\n",
    "\n",
    "    return int(ls_iter), int(s_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "files = os.listdir(surrogate_data_folder)\n",
    "\n",
    "data_results = {}\n",
    "static_results = {}\n",
    "\n",
    "# extract static result\n",
    "for LS in [100, 500, 1000]: \n",
    "    \n",
    "    LS_files = [ result for result in os.listdir(surrogate_static_folder) if f'LS_{LS}-' in result ]\n",
    "\n",
    "    for f in LS_files:\n",
    "        filepath = os.path.join(surrogate_static_folder, f)\n",
    "\n",
    "        with open(filepath, 'r') as ff:\n",
    "\n",
    "            max_score = 0.\n",
    "            scores = []\n",
    "\n",
    "            for line in ff.readlines()[1:ILS+1]:\n",
    "                cscore = float(line.split(';')[-1])\n",
    "\n",
    "                if cscore > max_score:\n",
    "                    max_score = cscore\n",
    "\n",
    "                scores.append(max_score)\n",
    "\n",
    "        if LS not in static_results:\n",
    "            static_results[LS] = []\n",
    "            \n",
    "        static_results[LS].append(scores)\n",
    "\n",
    "# extract all other results\n",
    "for i in range(1, N_RUNS + 1):\n",
    "    for run in [ f for f in files if f\"RUN_{i}_performance\" in f ]:\n",
    "        \n",
    "        key = extract_params(run)\n",
    "        if key not in data_results:\n",
    "            data_results[key] = []\n",
    "\n",
    "        # need to extract the 50 first solution found from surrogate data file\n",
    "        data_file = os.path.join(surrogate_data_folder, run.replace('_performance.csv', ''))\n",
    "\n",
    "        max_score = 0.\n",
    "        scores = []\n",
    "        with open(data_file, 'r') as f:\n",
    "            lines = f.readlines()\n",
    "\n",
    "            for line in lines[1:SS + 1]:\n",
    "                cscore = float(line.split(';')[-1])\n",
    "\n",
    "                if cscore > max_score:\n",
    "                    max_score = cscore\n",
    "                scores.append(max_score)\n",
    "        \n",
    "        with open(os.path.join(surrogate_data_folder, run), 'r') as f:\n",
    "            lines = f.readlines()\n",
    "\n",
    "            for index, line in enumerate(lines[:ILS - SS]):\n",
    "                #print(f'{index} => {run}')\n",
    "                #print(line)\n",
    "                cscore = float(line.split(';')[-2])\n",
    "\n",
    "                if cscore > max_score:\n",
    "                    max_score = cscore\n",
    "\n",
    "                scores.append(max_score)\n",
    "\n",
    "        # append run data into dictionnary with its key\n",
    "        data_results[key].append(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 1) has 5 runs\n",
      "(100, 2) has 5 runs\n",
      "(500, 1) has 5 runs\n",
      "(500, 2) has 5 runs\n",
      "(1000, 1) has 5 runs\n",
      "(1000, 2) has 5 runs\n"
     ]
    }
   ],
   "source": [
    "for k in sorted(data_results.keys()):\n",
    "    print(f'{k} has {len(data_results[k])} runs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 has 5 runs\n",
      "500 has 4 runs\n",
      "1000 has 5 runs\n"
     ]
    }
   ],
   "source": [
    "for k in sorted(static_results.keys()):\n",
    "    print(f'{k} has {len(static_results[k])} runs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35984 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 39104 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 400x400 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_LS = 100\n",
    "\n",
    "for n_LS in [100, 500, 1000]:\n",
    "    fig = plt.figure(dpi=100, figsize=(4, 4))\n",
    "    ax = fig.add_subplot(111)\n",
    "    #ax.set_title(f'Surroggate - LS = {n_LS}')\n",
    "\n",
    "    # display static results\n",
    "    data = np.array(static_results[n_LS])\n",
    "\n",
    "    means = data.mean(axis=0)\n",
    "    confidences = (1.9 * data.std(axis=0)) / np.sqrt(data.shape[0])\n",
    "            \n",
    "    ax.plot(means, label=f'Classique')\n",
    "    ax.fill_between(np.arange(data.shape[1]), means-confidences, means+confidences, alpha=.25)\n",
    "\n",
    "    for k in sorted(data_results.keys()):\n",
    "        ls_iter, order = k\n",
    "\n",
    "        if ls_iter == n_LS:\n",
    "            \n",
    "            data = np.array(data_results[k])\n",
    "\n",
    "            means = data.mean(axis=0)\n",
    "            confidences = (1.9 * data.std(axis=0)) / np.sqrt(N_RUNS)\n",
    "            \n",
    "            ax.plot(means, label=f'Ordre {order}')\n",
    "            ax.fill_between(np.arange(1000), means-confidences, means+confidences, alpha=.25)\n",
    "\n",
    "    ax.grid()\n",
    "    ax.legend(loc=4)\n",
    "    ax.set_ylabel('AUC ROC')\n",
    "    ax.set_xlabel('Nombre d\\'évaluations')\n",
    "    ax.set_ylim(0.7, 0.82)\n",
    "    # ax.set_yscale('log')\n",
    "    fig.savefig(os.path.join(output_images_folder, f'out_performance_LS-{n_LS}.pdf'), format='pdf', bbox_inches='tight')\n",
    "    #plt.show()\n",
    "fig.clf()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15, 1000)\n",
      "(15, 1000)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 42612 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 400x400 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for o in [1, 2]:\n",
    "    fig = plt.figure(dpi=100, figsize=(4, 4))\n",
    "    ax = fig.add_subplot(111)\n",
    "    #ax.set_title(f'Surrogate - Order = {o}')\n",
    "\n",
    "    # display static results\n",
    "    data_list = []\n",
    "    for k in static_results:\n",
    "        data_list = data_list + static_results[k]\n",
    "        \n",
    "    data = np.array(data_list)\n",
    "\n",
    "    means = data.mean(axis=0)\n",
    "    confidences = (1.9 * data.std(axis=0)) / np.sqrt(data.shape[0])\n",
    "            \n",
    "    ax.plot(means, label=f'Classique')\n",
    "    ax.fill_between(np.arange(data.shape[1]), means-confidences, means+confidences, alpha=.25)\n",
    "\n",
    "    for k in sorted(data_results.keys()):\n",
    "        ls_iter, order = k\n",
    "\n",
    "        if o == order:\n",
    "            \n",
    "            data = np.array(data_results[k])\n",
    "\n",
    "            means = data.mean(axis=0)\n",
    "            confidences = (1.9 * data.std(axis=0)) / np.sqrt(N_RUNS)\n",
    "            \n",
    "            ax.plot(means, label=f'LS {ls_iter}')\n",
    "            ax.fill_between(np.arange(1000), means-confidences, means+confidences, alpha=.25)\n",
    "\n",
    "    ax.grid()\n",
    "    ax.legend(loc=4)\n",
    "    ax.set_ylabel('AUC ROC')\n",
    "    ax.set_xlabel('Nombre d\\'évaluations')\n",
    "    ax.set_ylim(0.7, 0.82)\n",
    "    # ax.set_yscale('log')\n",
    "    fig.savefig(os.path.join(output_images_folder, f'out_performance_SO-{o}.pdf'), format='pdf', bbox_inches='tight')\n",
    "    #plt.show()\n",
    "fig.clf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Curve of model training scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = os.listdir(surrogate_data_folder)\n",
    "\n",
    "data_results = {}\n",
    "for i in range(1, N_RUNS + 1):\n",
    "    for run in [ f for f in files if f\"RUN_{i}_performance\" in f ]:\n",
    "        \n",
    "        key = extract_params(run)\n",
    "        if key not in data_results:\n",
    "            data_results[key] = []\n",
    "\n",
    "        # need to extract the 50 first solution found from surrogate data file\n",
    "        data_file = os.path.join(surrogate_data_folder, run.replace('_performance.csv', ''))\n",
    "\n",
    "        scores = []\n",
    "        \n",
    "        with open(os.path.join(surrogate_data_folder, run), 'r') as f:\n",
    "            lines = f.readlines()\n",
    "\n",
    "            for index, line in enumerate(lines[:ILS - SS]):\n",
    "                #print(f'{index} => {run}')\n",
    "                #print(line)\n",
    "                scores.append(float(line.split(';')[4]))\n",
    "\n",
    "        # append run data into dictionnary with its key\n",
    "        data_results[key].append(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 52620 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 53564 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 400x400 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_LS = 100\n",
    "\n",
    "for n_LS in [100, 500, 1000]:\n",
    "    fig = plt.figure(dpi=100, figsize=(4, 4))\n",
    "    ax = fig.add_subplot(111)\n",
    "    #ax.set_title(f'Surrogate model - LS {n_LS}')\n",
    "\n",
    "    for k in sorted(data_results.keys()):\n",
    "        ls_iter, order = k\n",
    "\n",
    "        if ls_iter == n_LS:\n",
    "            \n",
    "            data = np.array(data_results[k])\n",
    "\n",
    "            means = data.mean(axis=0)\n",
    "            confidences = (1.9 * data.std(axis=0)) / np.sqrt(N_RUNS)\n",
    "            \n",
    "            ax.plot(means, label=f'order {order}')\n",
    "            ax.fill_between(np.arange(ILS - SS), means-confidences, means+confidences, alpha=.25)\n",
    "\n",
    "    ax.grid()\n",
    "    ax.legend(loc=4)\n",
    "    ax.set_ylabel('R²')\n",
    "    ax.set_xlabel('Nombre d\\'évaluations')\n",
    "    ax.set_ylim(0.5, 1)\n",
    "    ax.set_xticks([0, 150, 350, 550, 750, 950]) \n",
    "    ax.set_xticklabels([50, 200, 400, 600, 800, 1000]) \n",
    "    # ax.set_yscale('log')\n",
    "    fig.savefig(os.path.join(output_images_folder, f'model_scores_LS-{n_LS}.pdf'), format='pdf', bbox_inches='tight')\n",
    "    #plt.show()\n",
    "fig.clf()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 62368 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 400x400 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for o in [1, 2]:\n",
    "    fig = plt.figure(dpi=100, figsize=(4, 4))\n",
    "    ax = fig.add_subplot(111)\n",
    "    #ax.set_title(f'Surrogate model - order {o}')\n",
    "\n",
    "    for k in sorted(data_results.keys()):\n",
    "        ls_iter, order = k\n",
    "\n",
    "        if o == order:\n",
    "            \n",
    "            data = np.array(data_results[k])\n",
    "\n",
    "            means = data.mean(axis=0)\n",
    "            confidences = (1.9 * data.std(axis=0)) / np.sqrt(N_RUNS)\n",
    "            \n",
    "            ax.plot(means, label=f'LS {ls_iter}')\n",
    "            ax.fill_between(np.arange(ILS - SS), means-confidences, means+confidences, alpha=.25)\n",
    "\n",
    "    ax.grid()\n",
    "    ax.legend(loc=4)\n",
    "    ax.set_ylabel('R²')\n",
    "    ax.set_xlabel('Nombre d\\'évaluations')\n",
    "    ax.set_ylim(0.5, 1)\n",
    "    \n",
    "    ax.set_xticks([0, 150, 350, 550, 750, 950]) \n",
    "    ax.set_xticklabels([50, 200, 400, 600, 800, 1000]) \n",
    "    # ax.set_yscale('log')\n",
    "    fig.savefig(os.path.join(output_images_folder, f'model_scores_SO-{o}.pdf'), format='pdf', bbox_inches='tight')\n",
    "    #plt.show()\n",
    "fig.clf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Display more selected attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_folder = os.path.join(data_folder, 'results')\n",
    "results_file = os.path.join(results_folder, 'optimization_comparisons_attributes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_solutions = {}\n",
    "with open(results_file, 'r') as f:\n",
    "    for line in f.readlines():\n",
    "        order = line.split(';')[0].split('-')[5].replace('SO_', '')\n",
    "        ls_iter = line.split(';')[0].split('-')[3].replace('LS_', '')\n",
    "        key = (int(ls_iter), int(order))\n",
    "        data_line = line.split(';')[2].replace('[', '').replace(']', '')\n",
    "        binary_sol = list(map(int, data_line.split(' ')))\n",
    "\n",
    "        if key not in all_solutions:\n",
    "            all_solutions[key] = []\n",
    "\n",
    "        all_solutions[key].append(binary_sol)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Global occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "binary_sols = []\n",
    "binary_sol_orders = {}\n",
    "for k in all_solutions:\n",
    "    ls_iter, order = k\n",
    "\n",
    "    if order not in binary_sol_orders:\n",
    "        binary_sol_orders[order] = []\n",
    "\n",
    "    for sol in all_solutions[k]:\n",
    "        binary_sols.append(sol)\n",
    "\n",
    "        binary_sol_orders[order].append(sol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 800x400 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 800x400 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "binary_sum = np.sum(binary_sols, axis=0)\n",
    "\n",
    "# overall\n",
    "fig = plt.figure(dpi=100, figsize=(8, 4))\n",
    "ax = fig.add_axes([0,0,1,1])\n",
    "#ax.set_title('Surrogate - attributes choices')\n",
    "ax.bar(np.arange(1, len(binary_sum) + 1), binary_sum, label='All')\n",
    "ax.set_xticks(np.arange(1, len(binary_sum) + 1))\n",
    "ax.set_ylabel('Occurences')\n",
    "ax.set_xlabel('Index de l\\'attribut')\n",
    "ax.set_ylabel('Occurences', fontsize=14)\n",
    "ax.set_xlabel('Index de l\\'attribut', fontsize=14)\n",
    "fig.savefig(os.path.join(output_images_folder, f'attributes_occurences.pdf'), format='pdf', bbox_inches='tight')\n",
    "fig.clf()\n",
    "\n",
    "# for each order\n",
    "fig = plt.figure(dpi=100, figsize=(8, 4))\n",
    "ax = fig.add_axes([0,0,1,1])\n",
    "#ax.set_title('Surrogate - attributes choices depending of order')\n",
    "\n",
    "widths = [-0.2, 0.2]\n",
    "for order in [1, 2]:\n",
    "    binary_sum = np.sum(binary_sol_orders[order], axis=0)\n",
    "    ax.bar(np.arange(1, len(binary_sum) + 1) - widths[order - 1], binary_sum, 0.4, label=f'Ordre {order}')\n",
    "ax.set_xticks(np.arange(1, len(binary_sum) + 1))\n",
    "ax.set_ylabel('Occurences', fontsize=14)\n",
    "ax.set_xlabel('Index de l\\'attribut', fontsize=14)\n",
    "ax.legend(loc=2, fontsize=14)\n",
    "fig.savefig(os.path.join(output_images_folder, f'attributes_occurences_orders.pdf'), format='pdf', bbox_inches='tight')\n",
    "fig.clf()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.7 64-bit ('surrogate-optim')",
   "language": "python",
   "name": "python38764bitsurrogateoptime172a7b2c75d4817b747a7ff436f986e"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
