AML: 03 Deep Neural Network for CIFAR-10¶
Data and Libraries¶
In [1]:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms, utils
import matplotlib.pyplot as plt
import numpy, random
# set the PseudoRandom Generator Seeds for better reproducibility
# see here for more: https://pytorch.org/docs/stable/notes/randomness.html
torch.manual_seed(99)
random.seed(99)
numpy.random.seed(99)
# this 'device' will be used for training our model
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
Load the CIFAR10 dataset¶
Observe that we set shuffle=True, which means that data is randomized
In [2]:
input_size = 32*32*3 # images are 32x32 pixels with 3 channels
output_size = 10 # there are 10 classes
train_loader = torch.utils.data.DataLoader(
datasets.CIFAR10('../data', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])),
batch_size=64, shuffle=True)
test_loader = torch.utils.data.DataLoader(
datasets.CIFAR10('../data', train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])),
batch_size=1000, shuffle=True)
classNames= ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
In [3]:
# show some training images
def imshow(img, plot):
img = img / 2 + 0.5 # unnormalize
npimg = img.numpy() # convert from tensor
plot.imshow(numpy.transpose(npimg, (1, 2, 0)))
plt.figure(figsize=(8,3), dpi=200)
# fetch a batch of train images; RANDOM
image_batch, label_batch = next(iter(train_loader))
#imshow(torchvision.utils.make_grid(image_batch))
for i in range(20):
image = image_batch[i]
label = classNames[label_batch[i].item()]
plt.subplot(2, 10, i + 1)
#image, label = train_loader.dataset.__getitem__(i)
#plt.imshow(image.squeeze().numpy())
imshow(image, plt)
plt.axis('off')
plt.title(label)
plt.show()
A 2-hidden layer Fully Connected Neural Network¶
Helper functions for training and testing¶
In [4]:
# function to count number of parameters
def get_n_params(model):
np=0
for p in list(model.parameters()):
np += p.nelement()
return np
accuracy_list = []
# we pass a model object to this trainer, and it trains this model for one epoch
def train(epoch, model):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
# send to device
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % 100 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.item()))
def test(model):
model.eval()
test_loss = 0
correct = 0
for data, target in test_loader:
# send to device
data, target = data.to(device), target.to(device)
output = model(data)
test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss
pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
correct += pred.eq(target.data.view_as(pred)).cpu().sum().item()
test_loss /= len(test_loader.dataset)
accuracy = 100. * correct / len(test_loader.dataset)
accuracy_list.append(accuracy)
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(test_loader.dataset),
accuracy))
Defining the Fully Connected Network¶
In [15]:
class FC2Layer(nn.Module):
def __init__(self, input_size, output_size):
super(FC2Layer, self).__init__()
self.input_size = input_size
self.network = nn.Sequential(
nn.Linear(input_size, 200),
nn.ReLU(),
nn.Linear(200,100),
nn.ReLU(),
nn.Linear(100,60),
nn.ReLU(),
nn.Linear(60, output_size),
nn.LogSoftmax(dim=1)
)
def forward(self, x):
x = x.view(-1, self.input_size)
return self.network(x)
Train the Network¶
In [16]:
print("Training on ", device)
model_fnn = FC2Layer(input_size, output_size)
model_fnn.to(device)
optimizer = optim.SGD(model_fnn.parameters(), lr=0.1)
print('Number of parameters: {}'.format(get_n_params(model_fnn)))
for epoch in range(0, 10):
train(epoch, model_fnn)
test(model_fnn)
Show some predictions of the test network¶
In [5]:
def visualize_pred(img, pred_prob, real_label):
''' Function for viewing an image and it's predicted classes.
'''
#pred_prob = pred_prob.data.numpy().squeeze()
fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)
#ax1.imshow(img.numpy().squeeze())
imshow(img, ax1)
ax1.axis('off')
pred_label = numpy.argmax(pred_prob)
ax1.set_title([classNames[real_label], classNames[pred_label]])
ax2.barh(numpy.arange(10), pred_prob)
ax2.set_aspect(0.1)
ax2.set_yticks(numpy.arange(10))
ax2.set_yticklabels(classNames)
ax2.set_title('Prediction Probability')
ax2.set_xlim(0, 1.1)
plt.tight_layout()
In [18]:
model_fnn.to('cpu')
# fetch a batch of test images
image_batch, label_batch = next(iter(test_loader))
# Turn off gradients to speed up this part
with torch.no_grad():
log_pred_prob_batch = model_fnn(image_batch)
for i in range(10):
img = image_batch[i]
real_label = label_batch[i].item()
log_pred_prob = log_pred_prob_batch[i]
# Output of the network are log-probabilities, need to take exponential for probabilities
pred_prob = torch.exp(log_pred_prob).data.numpy().squeeze()
visualize_pred(img, pred_prob, real_label)
FCN for CIFAR10 with Dropout¶
In [6]:
class FC2LayerDropout(nn.Module):
def __init__(self, input_size, output_size):
super(FC2LayerDropout, self).__init__()
self.input_size = input_size
self.network = nn.Sequential(
nn.Linear(input_size, 200),
nn.Dropout(0.25),
nn.ReLU(),
nn.Linear(200, 100),
nn.Dropout(0.25),
nn.ReLU(),
nn.Linear(100,60),
nn.Dropout(0.25),
nn.ReLU(),
nn.Linear(60, output_size),
nn.LogSoftmax(dim=1)
)
def forward(self, x):
x = x.view(-1, self.input_size)
return self.network(x)
In [22]:
print("With Dropout Training on ", device)
model = FC2LayerDropout(input_size, output_size)
model.to(device)
optimizer = optim.SGD(model.parameters(), lr=0.1)
print('Number of parameters: {}'.format(get_n_params(model)))
for epoch in range(0, 10):
model.train() # model in training mode. Turns on dropout, batch-norm etc during training
train(epoch, model)
model.eval() # model in evaluation mode. Turn off dropout, batch-norm etc during validation/testing
test(model)
In [23]:
model.to('cpu')
# fetch a batch of test images
image_batch, label_batch = next(iter(test_loader))
# Turn off gradients to speed up this part
with torch.no_grad():
log_pred_prob_batch = model(image_batch)
for i in range(10):
img = image_batch[i]
real_label = label_batch[i].item()
log_pred_prob = log_pred_prob_batch[i]
# Output of the network are log-probabilities, need to take exponential for probabilities
pred_prob = torch.exp(log_pred_prob).data.numpy().squeeze()
visualize_pred(img, pred_prob, real_label)
Does the Fully Connected Network use "Visual Information" ?¶
In [24]:
fixed_perm = torch.randperm(3072) # Fix a permutation of the image pixels; We apply the same permutation to all images
# show some training images
plt.figure(figsize=(8, 8))
# fetch a batch of train images; RANDOM
image_batch, label_batch = next(iter(train_loader))
for i in range(6):
image = image_batch[i]
image_perm = image.view(-1, 32*32*3).clone()
image_perm = image_perm[:, fixed_perm]
image_perm = image_perm.view(3, 32, 32)
label = label_batch[i].item()
plt.subplot(3,4 , 2*i + 1)
#image, label = train_loader.dataset.__getitem__(i)
#plt.imshow(image.squeeze().numpy())
imshow(image, plt)
plt.axis('off')
plt.title(classNames[label])
plt.subplot(3, 4, 2*i+2)
#plt.imshow(image_perm.squeeze().numpy())
imshow(image_perm, plt)
plt.axis('off')
plt.title(classNames[label])
In [25]:
accuracy_list = []
def scramble_train(epoch, model, perm=torch.arange(0, 3072).long()):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
# send to device
data, target = data.to(device), target.to(device)
# permute pixels
data = data.view(-1, 32*32*3)
data = data[:, perm]
data = data.view(-1, 3, 32, 32)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % 100 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.item()))
def scramble_test(model, perm=torch.arange(0, 3072).long()):
model.eval()
test_loss = 0
correct = 0
for data, target in test_loader:
# send to device
data, target = data.to(device), target.to(device)
# permute pixels
data = data.view(-1, 32*32*3)
data = data[:, perm]
data = data.view(-1, 3, 32, 32)
output = model(data)
test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss
pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
correct += pred.eq(target.data.view_as(pred)).cpu().sum().item()
test_loss /= len(test_loader.dataset)
accuracy = 100. * correct / len(test_loader.dataset)
accuracy_list.append(accuracy)
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(test_loader.dataset),
accuracy))
In [26]:
print("Training on ", device)
model_fnn_2 = FC2Layer(input_size, output_size)
model_fnn_2.to(device)
optimizer = optim.SGD(model_fnn_2.parameters(), lr=0.1)
print('Number of parameters: {}'.format(get_n_params(model_fnn_2)))
for epoch in range(0, 10):
scramble_train(epoch, model_fnn_2, fixed_perm)
scramble_test(model_fnn_2, fixed_perm)
In [27]:
model_fnn_2.to('cpu')
# fetch a batch of test images
image_batch, label_batch = next(iter(test_loader))
image_batch_scramble = image_batch.view(-1, 32*32*3)
image_batch_scramble = image_batch_scramble[:, fixed_perm]
image_batch_scramble = image_batch_scramble.view(-1, 3, 32, 32)
# Turn off gradients to speed up this part
with torch.no_grad():
log_pred_prob_batch = model_fnn_2(image_batch_scramble)
for i in range(10):
img = image_batch[i]
img_perm = image_batch_scramble[i]
real_label = label_batch[i].item()
log_pred_prob = log_pred_prob_batch[i]
# Output of the network are log-probabilities, need to take exponential for probabilities
pred_prob = torch.exp(log_pred_prob).data.numpy().squeeze()
visualize_pred(img_perm, pred_prob, real_label)
Better optimizer parameters¶
Smaller learning rate + momentum
In [31]:
print("Training on ", device)
model_fnn = FC2Layer(input_size, output_size)
model_fnn.to(device)
#optimizer = optim.SGD(model_fnn.parameters(), lr=0.01, momentum=0.5)
optimizer = optim.Adam(model_fnn.parameters())
print('Number of parameters: {}'.format(get_n_params(model_fnn)))
for epoch in range(0, 10):
train(epoch, model_fnn)
test(model_fnn)
In [33]:
model_fnn.to('cpu')
# fetch a batch of test images
image_batch, label_batch = next(iter(test_loader))
# Turn off gradients to speed up this part
with torch.no_grad():
log_pred_prob_batch = model_fnn(image_batch)
for i in range(10):
img = image_batch[i]
real_label = label_batch[i].item()
log_pred_prob = log_pred_prob_batch[i]
# Output of the network are log-probabilities, need to take exponential for probabilities
pred_prob = torch.exp(log_pred_prob).data.numpy().squeeze()
visualize_pred(img, pred_prob, real_label)
Better Optimizer + Dropout¶
In [8]:
print("Training on ", device)
model_fnn = FC2LayerDropout(input_size, output_size)
model_fnn.to(device)
#optimizer = optim.SGD(model_fnn.parameters(), lr=0.01, momentum=0.5)
optimizer = optim.Adam(model_fnn.parameters())
print('Number of parameters: {}'.format(get_n_params(model_fnn)))
for epoch in range(0, 10):
train(epoch, model_fnn)
test(model_fnn)
In [9]:
model_fnn.to('cpu')
# fetch a batch of test images
image_batch, label_batch = next(iter(test_loader))
# Turn off gradients to speed up this part
with torch.no_grad():
log_pred_prob_batch = model_fnn(image_batch)
for i in range(10):
img = image_batch[i]
real_label = label_batch[i].item()
log_pred_prob = log_pred_prob_batch[i]
# Output of the network are log-probabilities, need to take exponential for probabilities
pred_prob = torch.exp(log_pred_prob).data.numpy().squeeze()
visualize_pred(img, pred_prob, real_label)
