Parcourir la source

Add of gan for image synthesis

Jérôme BUISINE il y a 5 ans
Parent
commit
b89725ffd3
63 fichiers modifiés avec 378 ajouts et 1 suppressions
  1. 3 0
      .gitignore
  2. 20 0
      Dockerfile
  3. 10 0
      Makefile
  4. 1 0
      ganAtariImage.py
  5. 184 0
      ganSynthesisImage.py
  6. 73 0
      noise_gan.ipynb
  7. 3 1
      requirements.txt
  8. BIN
      synthesis_images/images/SdB2_00900.png
  9. BIN
      synthesis_images/images/SdB2_00910.png
  10. BIN
      synthesis_images/images/SdB2_00920.png
  11. BIN
      synthesis_images/images/SdB2_00930.png
  12. BIN
      synthesis_images/images/SdB2_00940.png
  13. BIN
      synthesis_images/images/SdB2_00950.png
  14. BIN
      synthesis_images/images/SdB2_D_00900.png
  15. BIN
      synthesis_images/images/SdB2_D_00910.png
  16. BIN
      synthesis_images/images/SdB2_D_00920.png
  17. BIN
      synthesis_images/images/SdB2_D_00930.png
  18. BIN
      synthesis_images/images/SdB2_D_00940.png
  19. BIN
      synthesis_images/images/SdB2_D_00950.png
  20. BIN
      synthesis_images/images/appartAopt_00850.png
  21. BIN
      synthesis_images/images/appartAopt_00860.png
  22. BIN
      synthesis_images/images/appartAopt_00870.png
  23. BIN
      synthesis_images/images/appartAopt_00880.png
  24. BIN
      synthesis_images/images/appartAopt_00890.png
  25. BIN
      synthesis_images/images/appartAopt_00900.png
  26. BIN
      synthesis_images/images/bureau1_10000.png
  27. BIN
      synthesis_images/images/bureau1_9750.png
  28. BIN
      synthesis_images/images/bureau1_9800.png
  29. BIN
      synthesis_images/images/bureau1_9850.png
  30. BIN
      synthesis_images/images/bureau1_9900.png
  31. BIN
      synthesis_images/images/bureau1_9950.png
  32. BIN
      synthesis_images/images/cendrierIUT2_01180.png
  33. BIN
      synthesis_images/images/cendrierIUT2_01240.png
  34. BIN
      synthesis_images/images/cendrierIUT2_01300.png
  35. BIN
      synthesis_images/images/cendrierIUT2_01360.png
  36. BIN
      synthesis_images/images/cendrierIUT2_01420.png
  37. BIN
      synthesis_images/images/cendrierIUT2_01480.png
  38. BIN
      synthesis_images/images/cuisine01_01150.png
  39. BIN
      synthesis_images/images/cuisine01_01160.png
  40. BIN
      synthesis_images/images/cuisine01_01170.png
  41. BIN
      synthesis_images/images/cuisine01_01180.png
  42. BIN
      synthesis_images/images/cuisine01_01190.png
  43. BIN
      synthesis_images/images/cuisine01_01200.png
  44. BIN
      synthesis_images/images/echecs09750.png
  45. BIN
      synthesis_images/images/echecs09800.png
  46. BIN
      synthesis_images/images/echecs09850.png
  47. BIN
      synthesis_images/images/echecs09900.png
  48. BIN
      synthesis_images/images/echecs09950.png
  49. BIN
      synthesis_images/images/echecs10000.png
  50. BIN
      synthesis_images/images/pnd_39750.png
  51. BIN
      synthesis_images/images/pnd_39800.png
  52. BIN
      synthesis_images/images/pnd_39850.png
  53. BIN
      synthesis_images/images/pnd_39900.png
  54. BIN
      synthesis_images/images/pnd_39950.png
  55. BIN
      synthesis_images/images/pnd_40000.png
  56. BIN
      synthesis_images/images/selles_envir02850.png
  57. BIN
      synthesis_images/images/selles_envir02900.png
  58. BIN
      synthesis_images/images/selles_envir02950.png
  59. BIN
      synthesis_images/images/selles_envir03000.png
  60. BIN
      synthesis_images/images/selles_envir03050.png
  61. BIN
      synthesis_images/images/selles_envir03100.png
  62. 35 0
      synthesis_images/prepare_data.py
  63. 49 0
      tensorboard.ipynb

+ 3 - 0
.gitignore

@@ -1,3 +1,6 @@
 runs
 .python-version
 .ipynb_checkpoints
+
+# do not track blocks images
+synthesis_images/generated_blocks

+ 20 - 0
Dockerfile

@@ -0,0 +1,20 @@
+FROM pytorch/pytorch
+
+# Update and add usefull package
+RUN apt-get update
+RUN apt-get install -y libglib2.0-dev libsm-dev libxrender-dev libxext-dev 
+
+# Install JupyterLab 
+RUN pip install jupyterlab && jupyter serverextension enable --py jupyterlab
+
+COPY requirements.txt requirements.txt
+
+RUN pip install -r requirements.txt
+
+ENV LANG=C.UTF-8
+
+# Expose Jupyter port & cmd 
+EXPOSE 8888 
+EXPOSE 6006
+
+CMD jupyter-lab --ip=0.0.0.0 --port=8888 --no-browser --notebook-dir=/data --allow-root

+ 10 - 0
Makefile

@@ -0,0 +1,10 @@
+current_dir := $(shell pwd)
+
+build: 
+	sudo docker build . --tag=jupyterlab-gan
+
+run:
+	sudo docker run -it -p 8888:8888 -p 6006:6006 -v $(current_dir):/data --name=noise jupyterlab-gan
+
+clean:
+	sudo docker container rm noise

+ 1 - 0
ganAtariImage.py

@@ -139,6 +139,7 @@ if __name__ == "__main__":
     envs = [InputWrapper(gym.make(name)) for name in ('Breakout-v0', 'AirRaid-v0', 'Pong-v0')]
     input_shape = envs[0].observation_space.shape
 
+    print(input_shape)
     net_discr = Discriminator(input_shape=input_shape).to(device)
     net_gener = Generator(output_shape=input_shape).to(device)
     print(net_gener)

+ 184 - 0
ganSynthesisImage.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+import random
+import argparse
+import cv2
+import os
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from tensorboardX import SummaryWriter
+from PIL import Image
+
+import torchvision.utils as vutils
+
+import gym
+import gym.spaces
+
+import numpy as np
+
+log = gym.logger
+log.set_level(gym.logger.INFO)
+
+LATENT_VECTOR_SIZE = 100
+DISCR_FILTERS = 64
+GENER_FILTERS = 64
+BATCH_SIZE = 16
+
+# dimension input image will be rescaled
+IMAGE_SIZE = 64
+input_shape = (3, IMAGE_SIZE, IMAGE_SIZE)
+
+LEARNING_RATE = 0.0001
+REPORT_EVERY_ITER = 50
+SAVE_IMAGE_EVERY_ITER = 200
+MAX_ITERATION = 100000
+
+data_folder = 'synthesis_images/generated_blocks'
+
+class Discriminator(nn.Module):
+    def __init__(self, input_shape):
+        super(Discriminator, self).__init__()
+        # this pipe converges image into the single number
+        self.conv_pipe = nn.Sequential(
+            nn.Conv2d(in_channels=input_shape[0], out_channels=DISCR_FILTERS,
+                      kernel_size=4, stride=2, padding=1),
+            nn.ReLU(),
+            nn.Conv2d(in_channels=DISCR_FILTERS, out_channels=DISCR_FILTERS*2,
+                      kernel_size=4, stride=2, padding=1),
+            nn.BatchNorm2d(DISCR_FILTERS*2),
+            nn.ReLU(),
+            nn.Conv2d(in_channels=DISCR_FILTERS * 2, out_channels=DISCR_FILTERS * 4,
+                      kernel_size=4, stride=2, padding=1),
+            nn.BatchNorm2d(DISCR_FILTERS * 4),
+            nn.ReLU(),
+            nn.Conv2d(in_channels=DISCR_FILTERS * 4, out_channels=DISCR_FILTERS * 8,
+                      kernel_size=4, stride=2, padding=1),
+            nn.BatchNorm2d(DISCR_FILTERS * 8),
+            nn.ReLU(),
+            nn.Conv2d(in_channels=DISCR_FILTERS * 8, out_channels=1,
+                      kernel_size=4, stride=1, padding=0),
+            nn.Sigmoid()
+        )
+
+    def forward(self, x):
+        conv_out = self.conv_pipe(x)
+        return conv_out.view(-1, 1).squeeze(dim=1)
+
+
+class Generator(nn.Module):
+    def __init__(self, output_shape):
+        super(Generator, self).__init__()
+        # pipe deconvolves input vector into (3, 64, 64) image
+        self.pipe = nn.Sequential(
+            nn.ConvTranspose2d(in_channels=LATENT_VECTOR_SIZE, out_channels=GENER_FILTERS * 8,
+                               kernel_size=4, stride=1, padding=0),
+            nn.BatchNorm2d(GENER_FILTERS * 8),
+            nn.ReLU(),
+            nn.ConvTranspose2d(in_channels=GENER_FILTERS * 8, out_channels=GENER_FILTERS * 4,
+                               kernel_size=4, stride=2, padding=1),
+            nn.BatchNorm2d(GENER_FILTERS * 4),
+            nn.ReLU(),
+            nn.ConvTranspose2d(in_channels=GENER_FILTERS * 4, out_channels=GENER_FILTERS * 2,
+                               kernel_size=4, stride=2, padding=1),
+            nn.BatchNorm2d(GENER_FILTERS * 2),
+            nn.ReLU(),
+            nn.ConvTranspose2d(in_channels=GENER_FILTERS * 2, out_channels=GENER_FILTERS,
+                               kernel_size=4, stride=2, padding=1),
+            nn.BatchNorm2d(GENER_FILTERS),
+            nn.ReLU(),
+            nn.ConvTranspose2d(in_channels=GENER_FILTERS, out_channels=output_shape[0],
+                               kernel_size=4, stride=2, padding=1),
+            nn.Tanh()
+        )
+
+    def forward(self, x):
+        return self.pipe(x)
+
+# here we have to generate our batches from final or noisy synthesis images
+def iterate_batches(batch_size=BATCH_SIZE):
+
+    batch = []
+    images = os.listdir(data_folder)
+    nb_images = len(images)
+
+    while True:
+        i = random.randint(0, nb_images - 1)
+
+        img = Image.open(os.path.join(data_folder, images[i]))
+        img_arr = np.asarray(img)
+
+        new_obs = cv2.resize(img_arr, (IMAGE_SIZE, IMAGE_SIZE))
+        # transform (210, 160, 3) -> (3, 210, 160)
+        new_obs = np.moveaxis(new_obs, 2, 0)
+
+        batch.append(new_obs)
+
+        if len(batch) == batch_size:
+            # Normalising input between -1 to 1
+            batch_np = np.array(batch, dtype=np.float32) * 2.0 / 255.0 - 1.0
+            yield torch.tensor(batch_np)
+            batch.clear()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cuda", default=False, action='store_true', help="Enable cuda computation")
+    args = parser.parse_args()
+
+    device = torch.device("cuda" if args.cuda else "cpu")
+    #envs = [InputWrapper(gym.make(name)) for name in ('Breakout-v0', 'AirRaid-v0', 'Pong-v0')]
+
+    print(input_shape)
+    net_discr = Discriminator(input_shape=input_shape).to(device)
+    net_gener = Generator(output_shape=input_shape).to(device)
+    print(net_gener)
+
+    objective = nn.BCELoss()
+    gen_optimizer = optim.Adam(params=net_gener.parameters(), lr=LEARNING_RATE, betas=(0.5, 0.999))
+    dis_optimizer = optim.Adam(params=net_discr.parameters(), lr=LEARNING_RATE, betas=(0.5, 0.999))
+    writer = SummaryWriter()
+
+    gen_losses = []
+    dis_losses = []
+    iter_no = 0
+
+    true_labels_v = torch.ones(BATCH_SIZE, dtype=torch.float32, device=device)
+    fake_labels_v = torch.zeros(BATCH_SIZE, dtype=torch.float32, device=device)
+
+    for batch_v in iterate_batches():
+
+        # generate extra fake samples, input is 4D: batch, filters, x, y
+        gen_input_v = torch.FloatTensor(BATCH_SIZE, LATENT_VECTOR_SIZE, 1, 1).normal_(0, 1).to(device)
+
+        # There we get data
+        batch_v = batch_v.to(device)
+        gen_output_v = net_gener(gen_input_v)
+
+        # train discriminator
+        dis_optimizer.zero_grad()
+        dis_output_true_v = net_discr(batch_v)
+        dis_output_fake_v = net_discr(gen_output_v.detach())
+        dis_loss = objective(dis_output_true_v, true_labels_v) + objective(dis_output_fake_v, fake_labels_v)
+        dis_loss.backward()
+        dis_optimizer.step()
+        dis_losses.append(dis_loss.item())
+
+        # train generator
+        gen_optimizer.zero_grad()
+        dis_output_v = net_discr(gen_output_v)
+        gen_loss_v = objective(dis_output_v, true_labels_v)
+        gen_loss_v.backward()
+        gen_optimizer.step()
+        gen_losses.append(gen_loss_v.item())
+
+        iter_no += 1
+        if iter_no % REPORT_EVERY_ITER == 0:
+            log.info("Iter %d: gen_loss=%.3e, dis_loss=%.3e", iter_no, np.mean(gen_losses), np.mean(dis_losses))
+            writer.add_scalar("gen_loss", np.mean(gen_losses), iter_no)
+            writer.add_scalar("dis_loss", np.mean(dis_losses), iter_no)
+            gen_losses = []
+            dis_losses = []
+        if iter_no % SAVE_IMAGE_EVERY_ITER == 0:
+            writer.add_image("fake", vutils.make_grid(gen_output_v.data[:IMAGE_SIZE], normalize=True), iter_no)
+            writer.add_image("real", vutils.make_grid(batch_v.data[:IMAGE_SIZE], normalize=True), iter_no)

+ 73 - 0
noise_gan.ipynb

@@ -0,0 +1,73 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(3, 64, 64)\n",
+      "Generator(\n",
+      "  (pipe): Sequential(\n",
+      "    (0): ConvTranspose2d(100, 512, kernel_size=(4, 4), stride=(1, 1))\n",
+      "    (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
+      "    (2): ReLU()\n",
+      "    (3): ConvTranspose2d(512, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
+      "    (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
+      "    (5): ReLU()\n",
+      "    (6): ConvTranspose2d(256, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
+      "    (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
+      "    (8): ReLU()\n",
+      "    (9): ConvTranspose2d(128, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
+      "    (10): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
+      "    (11): ReLU()\n",
+      "    (12): ConvTranspose2d(64, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
+      "    (13): Tanh()\n",
+      "  )\n",
+      ")\n",
+      "INFO: Iter 50: gen_loss=2.939e+00, dis_loss=4.371e-01\n",
+      "INFO: Iter 100: gen_loss=5.404e+00, dis_loss=5.104e-02\n",
+      "INFO: Iter 150: gen_loss=6.753e+00, dis_loss=2.676e-02\n",
+      "INFO: Iter 200: gen_loss=6.347e+00, dis_loss=2.015e-02\n",
+      "INFO: Iter 250: gen_loss=6.294e+00, dis_loss=3.237e-01\n",
+      "INFO: Iter 300: gen_loss=5.793e+00, dis_loss=2.149e-01\n"
+     ]
+    }
+   ],
+   "source": [
+    "!python ganSynthesisImage.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

+ 3 - 1
requirements.txt

@@ -1,7 +1,9 @@
 gym
-gym[atari]
+atari_py
 torch
 torchvision
 tensorflow
 tensorboardX
 opencv-python
+ipfml
+Pillow

BIN
synthesis_images/images/SdB2_00900.png


BIN
synthesis_images/images/SdB2_00910.png


BIN
synthesis_images/images/SdB2_00920.png


BIN
synthesis_images/images/SdB2_00930.png


BIN
synthesis_images/images/SdB2_00940.png


BIN
synthesis_images/images/SdB2_00950.png


BIN
synthesis_images/images/SdB2_D_00900.png


BIN
synthesis_images/images/SdB2_D_00910.png


BIN
synthesis_images/images/SdB2_D_00920.png


BIN
synthesis_images/images/SdB2_D_00930.png


BIN
synthesis_images/images/SdB2_D_00940.png


BIN
synthesis_images/images/SdB2_D_00950.png


BIN
synthesis_images/images/appartAopt_00850.png


BIN
synthesis_images/images/appartAopt_00860.png


BIN
synthesis_images/images/appartAopt_00870.png


BIN
synthesis_images/images/appartAopt_00880.png


BIN
synthesis_images/images/appartAopt_00890.png


BIN
synthesis_images/images/appartAopt_00900.png


BIN
synthesis_images/images/bureau1_10000.png


BIN
synthesis_images/images/bureau1_9750.png


BIN
synthesis_images/images/bureau1_9800.png


BIN
synthesis_images/images/bureau1_9850.png


BIN
synthesis_images/images/bureau1_9900.png


BIN
synthesis_images/images/bureau1_9950.png


BIN
synthesis_images/images/cendrierIUT2_01180.png


BIN
synthesis_images/images/cendrierIUT2_01240.png


BIN
synthesis_images/images/cendrierIUT2_01300.png


BIN
synthesis_images/images/cendrierIUT2_01360.png


BIN
synthesis_images/images/cendrierIUT2_01420.png


BIN
synthesis_images/images/cendrierIUT2_01480.png


BIN
synthesis_images/images/cuisine01_01150.png


BIN
synthesis_images/images/cuisine01_01160.png


BIN
synthesis_images/images/cuisine01_01170.png


BIN
synthesis_images/images/cuisine01_01180.png


BIN
synthesis_images/images/cuisine01_01190.png


BIN
synthesis_images/images/cuisine01_01200.png


BIN
synthesis_images/images/echecs09750.png


BIN
synthesis_images/images/echecs09800.png


BIN
synthesis_images/images/echecs09850.png


BIN
synthesis_images/images/echecs09900.png


BIN
synthesis_images/images/echecs09950.png


BIN
synthesis_images/images/echecs10000.png


BIN
synthesis_images/images/pnd_39750.png


BIN
synthesis_images/images/pnd_39800.png


BIN
synthesis_images/images/pnd_39850.png


BIN
synthesis_images/images/pnd_39900.png


BIN
synthesis_images/images/pnd_39950.png


BIN
synthesis_images/images/pnd_40000.png


BIN
synthesis_images/images/selles_envir02850.png


BIN
synthesis_images/images/selles_envir02900.png


BIN
synthesis_images/images/selles_envir02950.png


BIN
synthesis_images/images/selles_envir03000.png


BIN
synthesis_images/images/selles_envir03050.png


BIN
synthesis_images/images/selles_envir03100.png


+ 35 - 0
synthesis_images/prepare_data.py

@@ -0,0 +1,35 @@
+from ipfml import processing
+from PIL import Image
+
+import shutil
+import os
+
+images_folder = "images"
+dest_folder = "generated_blocks"
+
+if os.path.exists(dest_folder):
+    # first remove folder if necessary
+    os.rmdir(os.path.abspath(os.path.join(os.getcwd(), dest_folder)))
+
+# create new images
+images = os.listdir(images_folder)
+
+if not os.path.exists(dest_folder):
+    os.makedirs(dest_folder)
+
+for img_path in images:
+
+    img = Image.open(os.path.join(images_folder, img_path))
+
+    blocks = processing.divide_in_blocks(img, (200, 200), pil=True)
+
+    for id, pil_block in enumerate(blocks):
+        img_name = img_path.split('/')[-1]
+        split_name = img_name.split('.')
+        block_name = split_name[0] + "_" + str(id) + "." + split_name[1]
+
+        dest_img_block = os.path.join(dest_folder, block_name)
+
+        pil_block.save(dest_img_block)
+
+

+ 49 - 0
tensorboard.ipynb

@@ -0,0 +1,49 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TensorBoard 1.12.2 at http://e0614dc78f89:6006 (Press CTRL+C to quit)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!tensorboard --logdir runs/Jan22_17-09-18_e0614dc78f89"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}