{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Generate realistic looking images of celebrity faces.\n\nBased on: https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html\nand https://www.kaggle.com/masrur007/dcgan-to-generate-new-faces-with-pytorch","metadata":{}},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"markdown","source":"At first we need to import the libraries. It is considered as standard imports.\n\n","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, transforms, models\nfrom torchvision.utils import make_grid\n\nfrom PIL import Image\nfrom IPython.display import display\nimport cv2\n\n\nimport glob\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport pickle as pkl\nimport matplotlib.pyplot as plt \n%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-21T10:04:10.39914Z","iopub.execute_input":"2021-11-21T10:04:10.399504Z","iopub.status.idle":"2021-11-21T10:04:12.117766Z","shell.execute_reply.started":"2021-11-21T10:04:10.399405Z","shell.execute_reply":"2021-11-21T10:04:12.117031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will be using this function mostly everywhere to run our experiments deterministically. Random functions of Numpy and Pandas will behave deterministically after this. To learn more about Deterministic Neural Networks please check out [this notebook](https://www.kaggle.com/bminixhofer/deterministic-neural-networks-using-pytorch)","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:04:12.119679Z","iopub.execute_input":"2021-11-21T10:04:12.119931Z","iopub.status.idle":"2021-11-21T10:04:12.128059Z","shell.execute_reply.started":"2021-11-21T10:04:12.119896Z","shell.execute_reply":"2021-11-21T10:04:12.12739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataloader and Visualization","metadata":{}},{"cell_type":"markdown","source":"Dataloader provides an iterable over the specified dataset by combining a dataset with a sampler. ","metadata":{}},{"cell_type":"code","source":"# Let's define the path\ndata_dir = '../input/celeba-dataset/img_align_celeba/'\n","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:04:12.129242Z","iopub.execute_input":"2021-11-21T10:04:12.129986Z","iopub.status.idle":"2021-11-21T10:04:12.135103Z","shell.execute_reply.started":"2021-11-21T10:04:12.129948Z","shell.execute_reply":"2021-11-21T10:04:12.134255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining the image transform rules\n'''\nThe augmentions are as follows, \n1. Resizing the image to 32x32 as the smaller the image the faster we can train \n2. Cropping from center with 32x32\n3. Chagning type to tensor\nwe won't be using normalize here as we will have to do that manually later for tanh activation function\n'''\n\ntransform_img = transforms.Compose([\n        transforms.Resize(32),  \n        transforms.CenterCrop(32),\n        transforms.ToTensor()\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:04:12.137196Z","iopub.execute_input":"2021-11-21T10:04:12.137571Z","iopub.status.idle":"2021-11-21T10:04:12.144974Z","shell.execute_reply.started":"2021-11-21T10:04:12.137537Z","shell.execute_reply":"2021-11-21T10:04:12.144189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataloader(batch_size, data_dir, transforms):\n    \"\"\"\n    Batch the neural network data using DataLoader\n    :param batch_size: The size of each batch; the number of images in a batch\n    :param data_dir: Directory where image data is located\n    :param transforms: data augmentations\n    :return: DataLoader with batched data\n    \"\"\"\n    \n    ds = datasets.ImageFolder(data_dir, transform_img)\n    data_loader = torch.utils.data.DataLoader(ds, batch_size=batch_size,num_workers= 4, shuffle=True)\n\n    return data_loader","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:04:12.146361Z","iopub.execute_input":"2021-11-21T10:04:12.146956Z","iopub.status.idle":"2021-11-21T10:04:12.153736Z","shell.execute_reply.started":"2021-11-21T10:04:12.146859Z","shell.execute_reply":"2021-11-21T10:04:12.153067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32 # Instead of individual samples, the data loader produces batched samples of given number\ntrain_loader = get_dataloader(batch_size,data_dir, transform_img)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:04:12.154942Z","iopub.execute_input":"2021-11-21T10:04:12.155408Z","iopub.status.idle":"2021-11-21T10:05:40.755699Z","shell.execute_reply.started":"2021-11-21T10:04:12.155372Z","shell.execute_reply":"2021-11-21T10:05:40.754948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_loader.dataset))","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:40.756813Z","iopub.execute_input":"2021-11-21T10:05:40.758869Z","iopub.status.idle":"2021-11-21T10:05:40.766098Z","shell.execute_reply.started":"2021-11-21T10:05:40.758838Z","shell.execute_reply":"2021-11-21T10:05:40.765402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's see if the train loader is working and sending us iterable images :) \n# Also note that we converted images to tensor above with ToTensor(). Now we need to convert back to numpy to plot them\n\n# obtain one batch of training images which means 32 images\ndataiter = iter(train_loader)\nimg, _ = dataiter.next() # _ for labels, Dataloader sends labels automatically as defined in Dataloader class.\n#However we don't need labels neither we assigned one :p so we put _ \n# plot the images in the batch, along with the corresponding labels\nfig = plt.figure(figsize=(30, 7))\nplot_size=30 # gonna plot 30 images only\nfor idx in np.arange(plot_size):\n    ax = fig.add_subplot(3, plot_size/3, idx+1, xticks=[], yticks=[])\n    npimg = img[idx].numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:40.767314Z","iopub.execute_input":"2021-11-21T10:05:40.767589Z","iopub.status.idle":"2021-11-21T10:05:42.600897Z","shell.execute_reply.started":"2021-11-21T10:05:40.767547Z","shell.execute_reply":"2021-11-21T10:05:42.599557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we need to scale our images as the output of a tanh activated generator will contain pixel values in a range from -1 to 1. So, we need to rescale our training images to a range of -1 to 1. ","metadata":{}},{"cell_type":"code","source":"def scale_images(x, max = 1.00 , min = -1.00):\n    x = x * (max - min) + min\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.602165Z","iopub.execute_input":"2021-11-21T10:05:42.602909Z","iopub.status.idle":"2021-11-21T10:05:42.608426Z","shell.execute_reply.started":"2021-11-21T10:05:42.602868Z","shell.execute_reply":"2021-11-21T10:05:42.60734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#let's check the scaling\nimg = img[5]\nprint('Before scaling min: ', img.min())\nprint('Before scaling max: ', img.max())\n\nscaled_img = scale_images(img)\n\nprint('After Scaling Min: ', scaled_img.min())\nprint('After Scaling Max: ', scaled_img.max())","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.612331Z","iopub.execute_input":"2021-11-21T10:05:42.613474Z","iopub.status.idle":"2021-11-21T10:05:42.653324Z","shell.execute_reply.started":"2021-11-21T10:05:42.613431Z","shell.execute_reply":"2021-11-21T10:05:42.652419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"now our min and max are in range of [-1,1]","metadata":{}},{"cell_type":"markdown","source":"# Defining Model","metadata":{}},{"cell_type":"markdown","source":"## Discriminator\nThe discriminator is a classifier that detects if the input samples are genuine or fabricated. As a result, the discriminator in a GAN is essentially a classifier. It attempts to discern between genuine data and data generated by the generator. It could utilize any network architecture suitable for the type of data it's classifying. Here I will be using convolutional classifier, only without any maxpooling layers. It is recommended to employ a deep network with normalization to deal with this kind of difficult data.","metadata":{}},{"cell_type":"code","source":"'''\nThe inputs to the discriminator are 32x32x3 tensor images\nThe output would be a single value that will indicate whether a given image is real or fake\n'''\ndef conv(in_channels, out_channels, kernel_size=4, stride=2, padding=1, batch_norm=True, bias = False):\n    layers = []\n    conv_layer = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, \n                           kernel_size=kernel_size, stride=stride, padding=padding)\n    #appending convolutional layer\n    layers.append(conv_layer)\n    #appending batch norm layer\n    if batch_norm:\n        layers.append(nn.BatchNorm2d(out_channels))\n        \n    return nn.Sequential(*layers)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.65476Z","iopub.execute_input":"2021-11-21T10:05:42.655078Z","iopub.status.idle":"2021-11-21T10:05:42.661118Z","shell.execute_reply.started":"2021-11-21T10:05:42.65504Z","shell.execute_reply":"2021-11-21T10:05:42.660337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Discriminator(nn.Module):\n\n    def __init__(self, conv_dim):\n        \"\"\"\n        Initializing the Discriminator Module\n        :param conv_dim: The depth of the first convolutional layer based on which we will create the  next ones where next  layer depth = 2 * previous layer depth\n        \"\"\"\n        super(Discriminator, self).__init__()\n\n        # complete init function\n        self.conv_dim = conv_dim\n        \n        \n        self.conv1 = conv(3, conv_dim, batch_norm=False)  \n        self.conv2 = conv(conv_dim, conv_dim*2)           \n        self.conv3 = conv(conv_dim*2, conv_dim*4)\n        self.conv4 = conv(conv_dim*4, conv_dim*8)\n        self.fc = nn.Linear(conv_dim*4*4*2, 1)\n\n    def forward(self, x):\n        \"\"\"\n        Forward propagation of the neural network\n        :param x: The input to the neural network     \n        :return: Discriminator logits; the output of the neural network\n        \"\"\"\n        # define feedforward behavior\n        x = F.leaky_relu(self.conv1(x), 0.2)\n        x = F.leaky_relu(self.conv2(x), 0.2)\n        x = F.leaky_relu(self.conv3(x), 0.2)\n        x = F.leaky_relu(self.conv4(x), 0.2)\n        \n        x = x.view(-1, self.conv_dim*4*2*4)\n        \n        x = self.fc(x)\n        \n        \n        return x\n","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.662784Z","iopub.execute_input":"2021-11-21T10:05:42.663417Z","iopub.status.idle":"2021-11-21T10:05:42.674212Z","shell.execute_reply.started":"2021-11-21T10:05:42.663352Z","shell.execute_reply":"2021-11-21T10:05:42.673243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generator \n\nThe name,itself is self explanatory. The generator component of a GAN learns to generate fake data by incorporating discriminator feedback. It learns to manipulate the discriminator so that its output is classified as real. The generator should upsample an input and create a new image with the same dimensions as our training data (28x28x3). This should mostly consist of transpose convolutional layers with normalzed output. ","metadata":{}},{"cell_type":"code","source":"def deconv(in_channels, out_channels, kernel_size=4, stride=2, padding=1, batch_norm=True, bias = False):\n    layers = []\n    \n    # append transpose conv layer -- we are not using bias terms in conv layers\n    layers.append(nn.ConvTranspose2d(in_channels, out_channels, kernel_size, stride, padding))\n    \n    # optional batch norm layer\n    if batch_norm:\n        layers.append(nn.BatchNorm2d(out_channels))\n        \n    return nn.Sequential(*layers)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.675697Z","iopub.execute_input":"2021-11-21T10:05:42.675968Z","iopub.status.idle":"2021-11-21T10:05:42.685968Z","shell.execute_reply.started":"2021-11-21T10:05:42.675933Z","shell.execute_reply":"2021-11-21T10:05:42.685172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Generator(nn.Module):\n    \n    def __init__(self, z_size, conv_dim):\n        \"\"\"\n        Initialize the Generator Module\n        :param z_size: The length of the input latent vector, z\n        :param conv_dim: The depth of the inputs to the *last* transpose convolutional layer\n        \"\"\"\n        super(Generator, self).__init__()\n        self.conv_dim = conv_dim\n        \n        self.fc = nn.Linear(z_size, conv_dim*4*4*4)\n        # complete init function\n        \n        self.de_conv1 = deconv(conv_dim*4, conv_dim*2)\n        self.de_conv2 = deconv(conv_dim*2, conv_dim)\n        self.de_conv3 = deconv(conv_dim, 3, 4, batch_norm=False )\n        \n        self.dropout = nn.Dropout(0.3)\n        \n        \n    def forward(self, x):\n        \"\"\"\n        Forward propagation of the neural network\n        :param x: The input to the neural network     \n        :return: A 32x32x3 Tensor image as output\n        \"\"\"\n        # define feedforward behavior\n        x = self.fc(x)\n        x = self.dropout(x)\n        \n        x = x.view(-1, self.conv_dim*4, 4, 4)\n        \n        x = F.relu(self.de_conv1(x))\n        x = F.relu(self.de_conv2(x))\n        x = self.de_conv3(x)\n        x = F.tanh(x)\n        \n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.68756Z","iopub.execute_input":"2021-11-21T10:05:42.68816Z","iopub.status.idle":"2021-11-21T10:05:42.699787Z","shell.execute_reply.started":"2021-11-21T10:05:42.688132Z","shell.execute_reply":"2021-11-21T10:05:42.698901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To help the models converge, we should initialize the weights of the convolutional and linear layers in your model. From the original [DCGAN paper](https://arxiv.org/pdf/1511.06434.pdf), they say:\n> All weights were initialized from a zero-centered Normal distribution with standard deviation 0.02.\n\n","metadata":{}},{"cell_type":"code","source":"#Initializing the weights to a normal distribution, centered around 0, with a standard deviation of 0.02.\n\ndef weights_init_normal(m):\n    \"\"\"\n    :param m: A module or layer in a network    \n    \"\"\"\n    # like `Conv`, `BatchNorm2d`, `Linear`, etc.\n    classname = m.__class__.__name__\n    \n    #  initial weights to convolutional and linear layers\n    if (classname.find('Conv') != -1 or classname.find('Linear') != -1):\n        nn.init.normal(m.weight.data, 0.0, 0.2)\n        \n    if hasattr(m, 'bias') and m.bias is not None:\n        nn.init.constant(m.bias.data, 0.0)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.702658Z","iopub.execute_input":"2021-11-21T10:05:42.702918Z","iopub.status.idle":"2021-11-21T10:05:42.711734Z","shell.execute_reply.started":"2021-11-21T10:05:42.70288Z","shell.execute_reply":"2021-11-21T10:05:42.710938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model hyperparameters\nd_conv_dim = 64\ng_conv_dim = 128\nz_size = 100\n# building discriminator and generator from the classes defined above\ndiscriminator = Discriminator(d_conv_dim)\ngenerator = Generator(z_size=z_size, conv_dim=g_conv_dim)\n\n# initialize model weights\ndiscriminator.apply(weights_init_normal)\ngenerator.apply(weights_init_normal)\nprint(\"done\")","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.713228Z","iopub.execute_input":"2021-11-21T10:05:42.713607Z","iopub.status.idle":"2021-11-21T10:05:42.815407Z","shell.execute_reply.started":"2021-11-21T10:05:42.713517Z","shell.execute_reply":"2021-11-21T10:05:42.814542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's look at our discriminator model\nprint(discriminator)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.816946Z","iopub.execute_input":"2021-11-21T10:05:42.81723Z","iopub.status.idle":"2021-11-21T10:05:42.822034Z","shell.execute_reply.started":"2021-11-21T10:05:42.81719Z","shell.execute_reply":"2021-11-21T10:05:42.821228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's look at our generator model\nprint(generator)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.823468Z","iopub.execute_input":"2021-11-21T10:05:42.823996Z","iopub.status.idle":"2021-11-21T10:05:42.83293Z","shell.execute_reply.started":"2021-11-21T10:05:42.823955Z","shell.execute_reply":"2021-11-21T10:05:42.831994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"use_gpu = torch.cuda.is_available()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.834697Z","iopub.execute_input":"2021-11-21T10:05:42.835201Z","iopub.status.idle":"2021-11-21T10:05:42.892152Z","shell.execute_reply.started":"2021-11-21T10:05:42.835164Z","shell.execute_reply":"2021-11-21T10:05:42.891369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"first, let's define essentials to train the model. ","metadata":{}},{"cell_type":"markdown","source":"### Optimizers\nTo learn about adam optimizer please check [this blog](https://towardsdatascience.com/adam-latest-trends-in-deep-learning-optimization-6be9a291375c). I found it helpful for me. :) ","metadata":{}},{"cell_type":"code","source":"lr = 0.0002 #learning rate\nbeta1=0.5\nbeta2=0.999\n\n# optimizers for the discriminator D and generator G\ndiscriminator_optimizer = torch.optim.Adam(discriminator.parameters(), lr, (beta1, beta2)) # for discriminator\ngenerator_optimizer = torch.optim.Adam(generator.parameters(), lr, (beta1, beta2)) # for generator","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.893624Z","iopub.execute_input":"2021-11-21T10:05:42.894077Z","iopub.status.idle":"2021-11-21T10:05:42.902383Z","shell.execute_reply.started":"2021-11-21T10:05:42.894023Z","shell.execute_reply":"2021-11-21T10:05:42.90168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loss functions for generator\nThe generator's goal is to get the discriminator to think its generated images are real.","metadata":{}},{"cell_type":"code","source":"def real_loss(D_out, smooth = False):\n    '''Calculates how close discriminator outputs are to being real.\n       param, D_out: discriminator logits\n       return: real loss'''\n    batch_size = D_out.size(0)\n    \n    if smooth:\n        labels = torch.ones(batch_size)*0.9\n    else:\n        labels = torch.ones(batch_size) \n    \n    if use_gpu:\n        labels = labels.cuda()\n    \n    criterion = nn.BCEWithLogitsLoss()\n    loss = criterion(D_out.squeeze(), labels)\n    \n    return loss\n\ndef fake_loss(D_out):\n    '''Calculates how close discriminator outputs are to being fake.\n       param, D_out: discriminator logits\n       return: fake loss'''\n    batch_size = D_out.size(0)\n    labels = torch.zeros(batch_size)\n    \n    if use_gpu:\n        labels = labels.cuda()\n        \n    criterion = nn.BCEWithLogitsLoss()\n    loss = criterion(D_out.squeeze(), labels)\n    \n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.904562Z","iopub.execute_input":"2021-11-21T10:05:42.904815Z","iopub.status.idle":"2021-11-21T10:05:42.911967Z","shell.execute_reply.started":"2021-11-21T10:05:42.904783Z","shell.execute_reply":"2021-11-21T10:05:42.911118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This train function was provided in the udacity deep learning nanodegree course. It's great. ","metadata":{}},{"cell_type":"markdown","source":"Training the discriminator by alternating on real and fake images.\nThen the generator, which tries to trick the discriminator and should have an opposing loss function.","metadata":{}},{"cell_type":"code","source":"def train(D, G, n_epochs, train_on_gpu, print_every=50):\n    '''Trains adversarial networks for some number of epochs\n       param, D: the discriminator network\n       param, G: the generator network\n       param, n_epochs: number of epochs to train for\n       param, print_every: when to print and record the models' losses\n       return: D and G losses'''\n    \n    # move models to GPU\n    if train_on_gpu:\n        D.cuda()\n        G.cuda()\n\n    # keep track of loss and generated, \"fake\" samples\n    samples = []\n    losses = []\n\n    # Get some fixed data for sampling. These are images that are held\n    # constant throughout training, and allow us to inspect the model's performance\n    sample_size=16\n    fixed_z = np.random.uniform(-1, 1, size=(sample_size, z_size))\n    fixed_z = torch.from_numpy(fixed_z).float()\n    # move z to GPU if available\n    if train_on_gpu:\n        fixed_z = fixed_z.cuda()\n\n    # epoch training loop\n    for epoch in range(n_epochs):\n\n        # batch training loop\n        for batch_i, (real_images, _) in enumerate(train_loader):\n\n            batch_size = real_images.size(0)\n            real_images = scale_images(real_images)\n\n            # Train the discriminator on real and fake images\n            discriminator_optimizer.zero_grad()\n            \n            if train_on_gpu:\n                real_images = real_images.cuda()\n                \n            D_real = D(real_images)\n            d_real_loss = real_loss(D_real)      # Calculates log(D(x))\n\n            z = np.random.uniform(-1, 1, size=(batch_size, z_size))\n            z = torch.from_numpy(z).float()\n\n            if train_on_gpu:\n                z = z.cuda()\n                \n            fake_images = G(z)\n            \n            D_fake = D(fake_images)\n            d_fake_loss = fake_loss(D_fake)\n            \n            \n            d_loss = d_real_loss + d_fake_loss\n            d_loss.backward()\n            discriminator_optimizer.step()     \n\n            # 2. Train the generator with an adversarial loss\n            generator_optimizer.zero_grad()\n            \n            z = np.random.uniform(-1, 1, size=(batch_size, z_size))\n            z = torch.from_numpy(z).float()\n            \n            if train_on_gpu:\n                z = z.cuda()\n            \n            fake_images = G(z)\n            \n            D_fake = D(fake_images)\n            \n            g_loss = real_loss(D_fake)  # The Generator wishies to minimize the real_loss for it's fake output\n                                        # to trick the Discriminator\n        \n            g_loss.backward()\n            generator_optimizer.step()\n            \n            # Print some loss stats\n            if batch_i % print_every == 0:\n                # append discriminator loss and generator loss\n                losses.append((d_loss.item(), g_loss.item()))\n                # print discriminator and generator loss\n                print('Epoch [{:5d}/{:5d}] | d_loss: {:6.4f} | g_loss: {:6.4f}'.format(\n                        epoch+1, n_epochs, d_loss.item(), g_loss.item()))\n\n\n        ## AFTER EACH EPOCH##    \n        # this code assumes your generator is named G, feel free to change the name\n        # generate and save sample, fake images\n        G.eval() # for generating samples\n        samples_z = G(fixed_z)\n        samples.append(samples_z)\n        G.train() # back to training mode\n\n    # Save training generator samples\n    with open('train_samples.pkl', 'wb') as f:\n        pkl.dump(samples, f)\n    \n    # finally return losses\n    return losses","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.913401Z","iopub.execute_input":"2021-11-21T10:05:42.913967Z","iopub.status.idle":"2021-11-21T10:05:42.930115Z","shell.execute_reply.started":"2021-11-21T10:05:42.913926Z","shell.execute_reply":"2021-11-21T10:05:42.929431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 10\nlosses = train(discriminator, generator, n_epochs=n_epochs, train_on_gpu = use_gpu)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T10:05:42.931487Z","iopub.execute_input":"2021-11-21T10:05:42.931973Z","iopub.status.idle":"2021-11-21T11:06:43.178474Z","shell.execute_reply.started":"2021-11-21T10:05:42.931937Z","shell.execute_reply":"2021-11-21T11:06:43.177654Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training loss","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots()\nlosses = np.array(losses)\nplt.plot(losses.T[0], label='Discriminator', alpha=0.5)\nplt.plot(losses.T[1], label='Generator', alpha=0.5)\nplt.title(\"Training Losses\")\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:06:43.180177Z","iopub.execute_input":"2021-11-21T11:06:43.18045Z","iopub.status.idle":"2021-11-21T11:06:43.432199Z","shell.execute_reply.started":"2021-11-21T11:06:43.180414Z","shell.execute_reply":"2021-11-21T11:06:43.431576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize the generated images","metadata":{}},{"cell_type":"code","source":"def view_samples(epoch, samples):\n    fig, axes = plt.subplots(figsize=(16,4), nrows=2, ncols=8, sharey=True, sharex=True)\n    for ax, img in zip(axes.flatten(), samples[epoch]):\n        img = img.detach().cpu().numpy()\n        img = np.transpose(img, (1, 2, 0))\n        img = ((img + 1)*255 / (2)).astype(np.uint8)\n        ax.xaxis.set_visible(False)\n        ax.yaxis.set_visible(False)\n        im = ax.imshow(img.reshape((32,32,3)))","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:06:43.433457Z","iopub.execute_input":"2021-11-21T11:06:43.43388Z","iopub.status.idle":"2021-11-21T11:06:43.440955Z","shell.execute_reply.started":"2021-11-21T11:06:43.433842Z","shell.execute_reply":"2021-11-21T11:06:43.440212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('train_samples.pkl', 'rb') as f:\n    samples = pkl.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:06:43.442224Z","iopub.execute_input":"2021-11-21T11:06:43.442483Z","iopub.status.idle":"2021-11-21T11:06:43.458026Z","shell.execute_reply.started":"2021-11-21T11:06:43.442447Z","shell.execute_reply":"2021-11-21T11:06:43.457383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = view_samples(-1, samples)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:06:43.459302Z","iopub.execute_input":"2021-11-21T11:06:43.459584Z","iopub.status.idle":"2021-11-21T11:06:44.337974Z","shell.execute_reply.started":"2021-11-21T11:06:43.459548Z","shell.execute_reply":"2021-11-21T11:06:44.337336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}