Update TD2 Deep Learning.ipynb

77b08b84 · Kabbaj Zineb · 8e9d8504 · 77b08b84
Commit 77b08b84 authored Nov 28, 2023 by Kabbaj Zineb
--- a/TD2 Deep Learning.ipynb
+++ b/TD2 Deep Learning.ipynb
@@ -1951,10 +1951,93 @@
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 107,
   "id": "572d824c",
   "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\ZINEB\\anaconda3\\lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
+      "  warnings.warn(msg)\n",
+      "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to C:\\Users\\ZINEB/.cache\\torch\\hub\\checkpoints\\resnet18-f37072fd.pth\n",
+      "100%|██████████| 44.7M/44.7M [00:10<00:00, 4.58MB/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/10\n",
+      "----------\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\ZINEB\\anaconda3\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:136: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
+      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "train Loss: 0.6227 Acc: 0.6680\n",
+      "val Loss: 0.3421 Acc: 0.8758\n",
+      "\n",
+      "Epoch 2/10\n",
+      "----------\n",
+      "train Loss: 0.7198 Acc: 0.7090\n",
+      "val Loss: 0.1835 Acc: 0.9477\n",
+      "\n",
+      "Epoch 3/10\n",
+      "----------\n",
+      "train Loss: 0.5178 Acc: 0.7746\n",
+      "val Loss: 0.1933 Acc: 0.9412\n",
+      "\n",
+      "Epoch 4/10\n",
+      "----------\n",
+      "train Loss: 0.4848 Acc: 0.8074\n",
+      "val Loss: 0.2029 Acc: 0.9281\n",
+      "\n",
+      "Epoch 5/10\n",
+      "----------\n",
+      "train Loss: 0.5139 Acc: 0.8074\n",
+      "val Loss: 0.2291 Acc: 0.9150\n",
+      "\n",
+      "Epoch 6/10\n",
+      "----------\n",
+      "train Loss: 0.5792 Acc: 0.8033\n",
+      "val Loss: 0.1806 Acc: 0.9412\n",
+      "\n",
+      "Epoch 7/10\n",
+      "----------\n",
+      "train Loss: 0.3262 Acc: 0.8607\n",
+      "val Loss: 0.2058 Acc: 0.9412\n",
+      "\n",
+      "Epoch 8/10\n",
+      "----------\n",
+      "train Loss: 0.3552 Acc: 0.8648\n",
+      "val Loss: 0.1882 Acc: 0.9412\n",
+      "\n",
+      "Epoch 9/10\n",
+      "----------\n",
+      "train Loss: 0.3464 Acc: 0.8484\n",
+      "val Loss: 0.1844 Acc: 0.9477\n",
+      "\n",
+      "Epoch 10/10\n",
+      "----------\n",
+      "train Loss: 0.3932 Acc: 0.8074\n",
+      "val Loss: 0.1730 Acc: 0.9412\n",
+      "\n",
+      "Training complete in 8m 30s\n",
+      "Best val Acc: 0.947712\n"
+     ]
+    }
+   ],
   "source": [
    "import copy\n",
    "import os\n",
@@ -2152,6 +2235,171 @@
    "Apply ther quantization (post and quantization aware) and evaluate impact on model size and accuracy."
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": 108,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#eval_model\n",
+    "\n",
+    "def eval_model(model, dataloader, criterion):\n",
+    "    model.eval()  # Set the model to evaluate mode\n",
+    "    running_loss = 0.0\n",
+    "    running_corrects = 0\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        for inputs, labels in dataloader:\n",
+    "            inputs = inputs.to(device)\n",
+    "            labels = labels.to(device)\n",
+    "\n",
+    "            outputs = model(inputs)\n",
+    "            _, preds = torch.max(outputs, 1)\n",
+    "            loss = criterion(outputs, labels)\n",
+    "\n",
+    "            running_loss += loss.item() * inputs.size(0)\n",
+    "            running_corrects += torch.sum(preds == labels.data)\n",
+    "\n",
+    "    loss = running_loss / len(dataloader.dataset)\n",
+    "    accuracy = running_corrects.double() / len(dataloader.dataset)\n",
+    "\n",
+    "    print(\"Evaluation Loss: {:.4f} Accuracy: {:.4f}\".format(loss, accuracy))\n",
+    "    return loss, accuracy\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 109,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\ZINEB\\anaconda3\\lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
+      "  warnings.warn(\n",
+      "c:\\Users\\ZINEB\\anaconda3\\lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
+      "  warnings.warn(msg)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/10\n",
+      "----------\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\ZINEB\\anaconda3\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:136: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
+      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "train Loss: 0.6425 Acc: 0.6680\n",
+      "val Loss: 0.3366 Acc: 0.9346\n",
+      "\n",
+      "Epoch 2/10\n",
+      "----------\n",
+      "train Loss: 0.4859 Acc: 0.7582\n",
+      "val Loss: 0.2422 Acc: 0.9412\n",
+      "\n",
+      "Epoch 3/10\n",
+      "----------\n",
+      "train Loss: 0.4214 Acc: 0.8279\n",
+      "val Loss: 0.2159 Acc: 0.9346\n",
+      "\n",
+      "Epoch 4/10\n",
+      "----------\n",
+      "train Loss: 0.3997 Acc: 0.8361\n",
+      "val Loss: 0.2607 Acc: 0.9085\n",
+      "\n",
+      "Epoch 5/10\n",
+      "----------\n",
+      "train Loss: 0.4037 Acc: 0.8115\n",
+      "val Loss: 0.2860 Acc: 0.8824\n",
+      "\n",
+      "Epoch 6/10\n",
+      "----------\n",
+      "train Loss: 0.3597 Acc: 0.8443\n",
+      "val Loss: 0.2590 Acc: 0.9085\n",
+      "\n",
+      "Epoch 7/10\n",
+      "----------\n",
+      "train Loss: 0.2971 Acc: 0.8770\n",
+      "val Loss: 0.2063 Acc: 0.9281\n",
+      "\n",
+      "Epoch 8/10\n",
+      "----------\n",
+      "train Loss: 0.3260 Acc: 0.8566\n",
+      "val Loss: 0.1808 Acc: 0.9412\n",
+      "\n",
+      "Epoch 9/10\n",
+      "----------\n",
+      "train Loss: 0.3064 Acc: 0.8730\n",
+      "val Loss: 0.1748 Acc: 0.9477\n",
+      "\n",
+      "Epoch 10/10\n",
+      "----------\n",
+      "train Loss: 0.2438 Acc: 0.9139\n",
+      "val Loss: 0.1790 Acc: 0.9477\n",
+      "\n",
+      "Training complete in 8m 28s\n",
+      "Best val Acc: 0.947712\n",
+      "Evaluation Loss: 0.1748 Accuracy: 0.9477\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(0.1747902406309579, tensor(0.9477, dtype=torch.float64))"
+      ]
+     },
+     "execution_count": 109,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Download a pre-trained ResNet18 model and freeze its weights\n",
+    "model = torchvision.models.resnet18(pretrained=True)\n",
+    "\n",
+    "# Replace the final fully connected layer with two layers and dropout\n",
+    "num_ftrs = model.fc.in_features\n",
+    "model.fc = nn.Sequential(\n",
+    "    nn.Linear(num_ftrs, 256),  # Add a middle layer with 256 units\n",
+    "    nn.ReLU(),  # ReLU activation for the middle layer\n",
+    "    nn.Dropout(0.5),  # Dropout with a probability of 0.5\n",
+    "    nn.Linear(256, 2),  # Output layer with 2 classes\n",
+    ")\n",
+    "\n",
+    "# Send the model to the GPU\n",
+    "model = model.to(device)\n",
+    "\n",
+    "# Set the loss function\n",
+    "criterion = nn.CrossEntropyLoss()\n",
+    "\n",
+    "# Observe that all parameters are being optimized\n",
+    "optimizer_all = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n",
+    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_all, step_size=7, gamma=0.1)\n",
+    "\n",
+    "# Train the model with the new architecture\n",
+    "model, epoch_time = train_model(model, criterion, optimizer_all, exp_lr_scheduler, num_epochs=10)\n",
+    "\n",
+    "# Evaluate the model on the test set\n",
+    "test_dataloader = torch.utils.data.DataLoader(\n",
+    "    image_datasets[\"val\"], batch_size=4, shuffle=False, num_workers=4\n",
+    ")\n",
+    "eval_model(model, test_dataloader, criterion)\n"
+   ]
+  },
  {
   "cell_type": "markdown",
   "id": "04a263f0",
 %% Cell type:markdown id:7edf7168 tags:
 # TD2: Deep learning
 %% Cell type:markdown id:fbb8c8df tags:
 In this TD, you must modify this notebook to answer the questions. To do this,
 1. Fork this repository
 2. Clone your forked repository on your local computer
 3. Answer the questions
 4. Commit and push regularly
 The last commit is due on Sunday, December 1, 11:59 PM. Later commits will not be taken into account.
 %% Cell type:markdown id:3d167a29 tags:
 Install and test PyTorch from  https://pytorch.org/get-started/locally.
 %% Cell type:code id: tags:
 ``` python
 import os
 os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'
 ```
 %% Cell type:code id:330a42f5 tags:
 ``` python
 %pip install torch torchvision
 ```
 %% Output
    Requirement already satisfied: torch in c:\users\zineb\anaconda3\lib\site-packages (2.1.0)
    Requirement already satisfied: torchvision in c:\users\zineb\anaconda3\lib\site-packages (0.16.0)
    Requirement already satisfied: typing-extensions in c:\users\zineb\anaconda3\lib\site-packages (from torch) (4.4.0)
    Requirement already satisfied: jinja2 in c:\users\zineb\anaconda3\lib\site-packages (from torch) (3.1.2)
    Requirement already satisfied: fsspec in c:\users\zineb\anaconda3\lib\site-packages (from torch) (2022.11.0)
    Requirement already satisfied: filelock in c:\users\zineb\anaconda3\lib\site-packages (from torch) (3.9.0)
    Requirement already satisfied: networkx in c:\users\zineb\anaconda3\lib\site-packages (from torch) (2.8.4)
    Requirement already satisfied: sympy in c:\users\zineb\anaconda3\lib\site-packages (from torch) (1.11.1)
    Requirement already satisfied: numpy in c:\users\zineb\anaconda3\lib\site-packages (from torchvision) (1.23.5)
    Requirement already satisfied: requests in c:\users\zineb\anaconda3\lib\site-packages (from torchvision) (2.28.1)
    Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in c:\users\zineb\anaconda3\lib\site-packages (from torchvision) (9.4.0)
    Requirement already satisfied: MarkupSafe>=2.0 in c:\users\zineb\anaconda3\lib\site-packages (from jinja2->torch) (2.1.1)
    Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\users\zineb\anaconda3\lib\site-packages (from requests->torchvision) (1.26.14)
    Requirement already satisfied: charset-normalizer<3,>=2 in c:\users\zineb\anaconda3\lib\site-packages (from requests->torchvision) (2.0.4)
    Requirement already satisfied: idna<4,>=2.5 in c:\users\zineb\anaconda3\lib\site-packages (from requests->torchvision) (3.4)
    Requirement already satisfied: certifi>=2017.4.17 in c:\users\zineb\anaconda3\lib\site-packages (from requests->torchvision) (2022.12.7)
    Requirement already satisfied: mpmath>=0.19 in c:\users\zineb\anaconda3\lib\site-packages (from sympy->torch) (1.2.1)
    Note: you may need to restart the kernel to use updated packages.
 %% Cell type:markdown id:0882a636 tags:
 To test run the following code
 %% Cell type:code id:b1950f0a tags:
 ``` python
 import torch
 N, D = 14, 10
 x = torch.randn(N, D).type(torch.FloatTensor)
 print(x)
 from torchvision import models
 alexnet = models.alexnet()
 print(alexnet)
 ```
 %% Output
    tensor([[-1.1210e+00,  5.2764e-01, -3.1968e-01,  2.2298e-02,  3.2028e-01,
              1.1693e+00,  9.9867e-03,  1.6576e+00, -6.4607e-01, -1.0916e+00],
            [ 1.0161e-01, -1.3867e-01, -1.1610e+00, -5.4409e-01,  3.9804e-01,
              4.3068e-01, -1.3733e+00,  6.4579e-01, -9.3711e-01, -6.2921e-01],
            [-4.5891e-01,  1.7762e+00,  3.5168e-01,  8.2529e-01, -3.6480e-01,
             -9.3685e-01,  8.2215e-01,  6.8467e-01, -4.3484e-01,  1.7282e+00],
            [ 2.3554e-01, -6.2146e-01, -9.5119e-01,  3.6604e-01, -5.5549e-02,
             -1.5742e-01,  8.4236e-01, -1.6707e+00,  3.5272e-01,  1.2580e-01],
            [-6.1302e-01, -7.8174e-02,  2.0755e+00, -5.7493e-01,  1.8069e+00,
             -1.1747e+00,  1.1533e+00,  4.4674e-01,  8.0904e-01,  1.2371e+00],
            [ 6.5255e-01, -2.4173e-01, -1.1272e-01, -8.6760e-01,  3.9370e-01,
              2.4600e-01, -1.2426e-01,  3.1234e-01, -4.4381e-01, -3.1786e-01],
            [-1.7306e+00,  8.6443e-01, -4.1809e-02, -1.3328e+00,  9.7420e-01,
             -4.8587e-01,  8.9359e-01, -3.0943e-01,  1.0975e+00, -1.5249e-03],
            [ 1.4104e+00, -1.3197e+00, -9.9384e-01, -1.0551e+00, -9.5739e-02,
              8.5214e-01,  5.9754e-01,  4.2689e-01,  4.4546e-01, -5.3021e-01],
            [ 7.9181e-01,  4.7276e-01,  1.1692e+00, -4.4760e-01, -4.8100e-01,
             -5.0203e-01,  1.3627e+00, -1.7923e-01,  7.2266e-01,  1.0586e-01],
            [-2.7925e-01, -2.4732e-01,  6.7349e-01, -9.2926e-01, -9.7715e-02,
              7.5156e-01,  5.2089e-01,  5.8953e-01, -2.2539e-01,  4.2665e-01],
            [ 2.9770e-01,  7.1523e-01, -5.1163e-01, -1.2523e+00, -2.9311e-01,
              6.6724e-01,  5.6068e-01,  8.2418e-02, -3.0311e-01,  1.3625e+00],
            [ 1.9347e-01, -1.9955e-01, -4.1287e-01, -1.3899e+00, -2.0153e-01,
              7.8712e-01, -1.1849e+00, -2.5764e-01, -2.4629e-01, -1.4975e-01],
            [ 1.7681e+00,  7.0654e-01, -2.1209e+00, -3.2242e-01, -1.7948e-01,
              8.7081e-01,  4.8530e-01, -7.2095e-01, -1.3229e+00,  1.9485e-01],
            [ 7.5603e-01, -3.8439e-01, -3.0080e-01,  5.0654e-01,  9.5246e-03,
              1.8669e+00, -2.6733e+00, -3.4223e-02, -9.7327e-01,  1.1582e-01]])
    AlexNet(
      (features): Sequential(
        (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
        (1): ReLU(inplace=True)
        (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
        (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
        (4): ReLU(inplace=True)
        (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
        (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
        (7): ReLU(inplace=True)
        (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
        (9): ReLU(inplace=True)
        (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
        (11): ReLU(inplace=True)
        (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
      )
      (avgpool): AdaptiveAvgPool2d(output_size=(6, 6))
      (classifier): Sequential(
        (0): Dropout(p=0.5, inplace=False)
        (1): Linear(in_features=9216, out_features=4096, bias=True)
        (2): ReLU(inplace=True)
        (3): Dropout(p=0.5, inplace=False)
        (4): Linear(in_features=4096, out_features=4096, bias=True)
        (5): ReLU(inplace=True)
        (6): Linear(in_features=4096, out_features=1000, bias=True)
      )
    )
 %% Cell type:markdown id:23f266da tags:
 ## Exercise 1: CNN on CIFAR10
 The goal is to apply a Convolutional Neural Net (CNN) model on the CIFAR10 image dataset and test the accuracy of the model on the basis of image classification. Compare the Accuracy VS the neural network implemented during TD1.
 Have a look at the following documentation to be familiar with PyTorch.
 https://pytorch.org/tutorials/beginner/pytorch_with_examples.html
 https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html
 %% Cell type:markdown id:4ba1c82d tags:
 You can test if GPU is available on your machine and thus train on it to speed up the process
 %% Cell type:code id:6e18f2fd tags:
 ``` python
 import torch
 # check if CUDA is available
 train_on_gpu = torch.cuda.is_available()
 if not train_on_gpu:
    print("CUDA is not available.  Training on CPU ...")
 else:
    print("CUDA is available!  Training on GPU ...")
 ```
 %% Output
    CUDA is not available.  Training on CPU ...
 %% Cell type:markdown id:5cf214eb tags:
 Next we load the CIFAR10 dataset
 %% Cell type:code id:462666a2 tags:
 ``` python
 import numpy as np
 from torchvision import datasets, transforms
 from torch.utils.data.sampler import SubsetRandomSampler
 # number of subprocesses to use for data loading
 num_workers = 0
 # how many samples per batch to load
 batch_size = 20
 # percentage of training set to use as validation
 valid_size = 0.2
 # convert data to a normalized torch.FloatTensor
 transform = transforms.Compose(
    [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
 )
 # choose the training and test datasets
 train_data = datasets.CIFAR10("data", train=True, download=True, transform=transform)
 test_data = datasets.CIFAR10("data", train=False, download=True, transform=transform)
 # obtain training indices that will be used for validation
 num_train = len(train_data)
 indices = list(range(num_train))
 np.random.shuffle(indices)
 split = int(np.floor(valid_size * num_train))
 train_idx, valid_idx = indices[split:], indices[:split]
 # define samplers for obtaining training and validation batches
 train_sampler = SubsetRandomSampler(train_idx)
 valid_sampler = SubsetRandomSampler(valid_idx)
 # prepare data loaders (combine dataset and sampler)
 train_loader = torch.utils.data.DataLoader(
    train_data, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers
 )
 valid_loader = torch.utils.data.DataLoader(
    train_data, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers
 )
 test_loader = torch.utils.data.DataLoader(
    test_data, batch_size=batch_size, num_workers=num_workers
 )
 # specify the image classes
 classes = [
    "airplane",
    "automobile",
    "bird",
    "cat",
    "deer",
    "dog",
    "frog",
    "horse",
    "ship",
    "truck",
 ]
 ```
 %% Output
    Files already downloaded and verified
    Files already downloaded and verified
 %% Cell type:markdown id:58ec3903 tags:
 CNN definition (this one is an example)
 %% Cell type:code id:317bf070 tags:
 ``` python
 import torch.nn as nn
 import torch.nn.functional as F
 # define the CNN architecture
 class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x
 # create a complete CNN
 model = Net()
 print(model)
 # move tensors to GPU if CUDA is available
 if train_on_gpu:
    model.cuda()
 ```
 %% Output
    Net(
      (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))
      (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))
      (fc1): Linear(in_features=400, out_features=120, bias=True)
      (fc2): Linear(in_features=120, out_features=84, bias=True)
      (fc3): Linear(in_features=84, out_features=10, bias=True)
    )
 %% Cell type:markdown id:a2dc4974 tags:
 Loss function and training using SGD (Stochastic Gradient Descent) optimizer
 %% Cell type:code id:4b53f229 tags:
 ``` python
 import torch.optim as optim
 criterion = nn.CrossEntropyLoss()  # specify loss function
 optimizer = optim.SGD(model.parameters(), lr=0.01)  # specify optimizer
 n_epochs = 30 # number of epochs to train the model
 train_loss_list = [] # list to store loss to visualize
 Valid_loss_list = []
 valid_loss_min = np.Inf  # track change in validation loss
 for epoch in range(n_epochs):
    # Keep track of training and validation loss
    train_loss = 0.0
    valid_loss = 0.0
    # Train the model
    model.train()
    for data, target in train_loader:
        # Move tensors to GPU if CUDA is available
        if train_on_gpu:
            data, target = data.cuda(), target.cuda()
        # Clear the gradients of all optimized variables
        optimizer.zero_grad()
        # Forward pass: compute predicted outputs by passing inputs to the model
        output = model(data)
        # Calculate the batch loss
        loss = criterion(output, target)
        # Backward pass: compute gradient of the loss with respect to model parameters
        loss.backward()
        # Perform a single optimization step (parameter update)
        optimizer.step()
        # Update training loss
        train_loss += loss.item() * data.size(0)
    # Validate the model
    model.eval()
    for data, target in valid_loader:
        # Move tensors to GPU if CUDA is available
        if train_on_gpu:
            data, target = data.cuda(), target.cuda()
        # Forward pass: compute predicted outputs by passing inputs to the model
        output = model(data)
        # Calculate the batch loss
        loss = criterion(output, target)
        # Update average validation loss
        valid_loss += loss.item() * data.size(0)
    # Calculate average losses
    train_loss = train_loss / len(train_loader)
    valid_loss = valid_loss / len(valid_loader)
    train_loss_list.append(train_loss)
    Valid_loss_list.append(valid_loss)
    # Print training/validation statistics
    print(
        "Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}".format(
            epoch, train_loss, valid_loss
        )
    )
    # Save model if validation loss has decreased
    if valid_loss <= valid_loss_min:
        print(
            "Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...".format(
                valid_loss_min, valid_loss
            )
        )
        torch.save(model.state_dict(), "model_cifar.pt")
        valid_loss_min = valid_loss
 ```
 %% Output
    Epoch: 0 	Training Loss: 30.219081 	Validation Loss: 29.449449
    Validation loss decreased (inf --> 29.449449).  Saving model ...
    Epoch: 1 	Training Loss: 28.063993 	Validation Loss: 26.581815
    Validation loss decreased (29.449449 --> 26.581815).  Saving model ...
    Epoch: 2 	Training Loss: 26.363633 	Validation Loss: 25.372685
    Validation loss decreased (26.581815 --> 25.372685).  Saving model ...
    Epoch: 3 	Training Loss: 24.926940 	Validation Loss: 24.959999
    Validation loss decreased (25.372685 --> 24.959999).  Saving model ...
    Epoch: 4 	Training Loss: 23.776435 	Validation Loss: 23.839217
    Validation loss decreased (24.959999 --> 23.839217).  Saving model ...
    Epoch: 5 	Training Loss: 22.797345 	Validation Loss: 23.305578
    Validation loss decreased (23.839217 --> 23.305578).  Saving model ...
    Epoch: 6 	Training Loss: 21.879690 	Validation Loss: 22.761072
    Validation loss decreased (23.305578 --> 22.761072).  Saving model ...
    Epoch: 7 	Training Loss: 21.059626 	Validation Loss: 22.490790
    Validation loss decreased (22.761072 --> 22.490790).  Saving model ...
    Epoch: 8 	Training Loss: 20.277269 	Validation Loss: 21.736995
    Validation loss decreased (22.490790 --> 21.736995).  Saving model ...
    Epoch: 9 	Training Loss: 19.577080 	Validation Loss: 21.602148
    Validation loss decreased (21.736995 --> 21.602148).  Saving model ...
    Epoch: 10 	Training Loss: 18.834896 	Validation Loss: 21.280448
    Validation loss decreased (21.602148 --> 21.280448).  Saving model ...
    Epoch: 11 	Training Loss: 18.216890 	Validation Loss: 21.327105
    Epoch: 12 	Training Loss: 17.571436 	Validation Loss: 21.276174
    Validation loss decreased (21.280448 --> 21.276174).  Saving model ...
    Epoch: 13 	Training Loss: 16.967960 	Validation Loss: 21.732455
    Epoch: 14 	Training Loss: 16.365070 	Validation Loss: 21.229552
    Validation loss decreased (21.276174 --> 21.229552).  Saving model ...
    Epoch: 15 	Training Loss: 15.752467 	Validation Loss: 20.502602
    Validation loss decreased (21.229552 --> 20.502602).  Saving model ...
    Epoch: 16 	Training Loss: 15.268329 	Validation Loss: 21.520030
    Epoch: 17 	Training Loss: 14.628543 	Validation Loss: 21.506971
    Epoch: 18 	Training Loss: 14.140162 	Validation Loss: 22.796522
    Epoch: 19 	Training Loss: 13.601739 	Validation Loss: 22.185051
    Epoch: 20 	Training Loss: 13.148037 	Validation Loss: 22.820398
    Epoch: 21 	Training Loss: 12.589797 	Validation Loss: 22.923253
    Epoch: 22 	Training Loss: 12.201906 	Validation Loss: 23.655233
    Epoch: 23 	Training Loss: 11.742708 	Validation Loss: 24.267373
    Epoch: 24 	Training Loss: 11.254362 	Validation Loss: 24.776800
    Epoch: 25 	Training Loss: 10.871113 	Validation Loss: 25.960069
    Epoch: 26 	Training Loss: 10.347627 	Validation Loss: 25.899857
    Epoch: 27 	Training Loss: 10.017751 	Validation Loss: 27.078486
    Epoch: 28 	Training Loss: 9.576065 	Validation Loss: 27.790080
    Epoch: 29 	Training Loss: 9.297409 	Validation Loss: 28.061284
 %% Cell type:markdown id:13e1df74 tags:
 Does overfit occur? If so, do an early stopping.
 %% Cell type:code id: tags:
 ``` python
 import matplotlib.pyplot as plt
 plt.plot(range(n_epochs), train_loss_list, label='Training Loss')
 plt.plot(range(n_epochs), Valid_loss_list, label='Validation Loss')
 plt.xlabel("Epoch")
 plt.ylabel("Loss")
 plt.title("Performance of Model 1")
 plt.legend()
 plt.show()
 ```
 %% Output
 %% Cell type:markdown id: tags:
 On peut détecter un overfitting en surveillant les performances du modèle sur les données
 d'entraînement et de test au fil du temps. Si les performances du modèle sur les données
 d'entraînement continuent de s'améliorer tandis que celles sur les données de test diminuent,
 cela indique un surapprentissage. Ici dans notre cas à partir de l'epoch 15, on remarque que la valeur de valid_loss commence à augmenter alors que train_loss diminue toujours.
 %% Cell type:markdown id:11df8fd4 tags:
 Now loading the model with the lowest validation loss value
 %% Cell type:code id:e93efdfc tags:
 ``` python
 model.load_state_dict(torch.load("./model_cifar.pt"))
 # track test loss
 test_loss = 0.0
 class_correct = list(0.0 for i in range(10))
 class_total = list(0.0 for i in range(10))
 model.eval()
 # iterate over test data
 for data, target in test_loader:
    # move tensors to GPU if CUDA is available
    if train_on_gpu:
        data, target = data.cuda(), target.cuda()
    # forward pass: compute predicted outputs by passing inputs to the model
    output = model(data)
    # calculate the batch loss
    loss = criterion(output, target)
    # update test loss
    test_loss += loss.item() * data.size(0)
    # convert output probabilities to predicted class
    _, pred = torch.max(output, 1)
    # compare predictions to true label
    correct_tensor = pred.eq(target.data.view_as(pred))
    correct = (
        np.squeeze(correct_tensor.numpy())
        if not train_on_gpu
        else np.squeeze(correct_tensor.cpu().numpy())
    )
    # calculate test accuracy for each object class
    for i in range(batch_size):
        label = target.data[i]
        class_correct[label] += correct[i].item()
        class_total[label] += 1
 # average test loss
 test_loss = test_loss / len(test_loader)
 print("Test Loss: {:.6f}\n".format(test_loss))
 for i in range(10):
    if class_total[i] > 0:
        print(
            "Test Accuracy of %5s: %2d%% (%2d/%2d)"
            % (
                classes[i],
                100 * class_correct[i] / class_total[i],
                np.sum(class_correct[i]),
                np.sum(class_total[i]),
            )
        )
    else:
        print("Test Accuracy of %5s: N/A (no training examples)" % (classes[i]))
 print(
    "\nTest Accuracy (Overall): %2d%% (%2d/%2d)"
    % (
        100.0 * np.sum(class_correct) / np.sum(class_total),
        np.sum(class_correct),
        np.sum(class_total),
    )
 )
 ```
 %% Output
    Test Loss: 21.195169
    Test Accuracy of airplane: 70% (702/1000)
    Test Accuracy of automobile: 77% (775/1000)
    Test Accuracy of  bird: 55% (555/1000)
    Test Accuracy of   cat: 36% (364/1000)
    Test Accuracy of  deer: 55% (557/1000)
    Test Accuracy of   dog: 54% (545/1000)
    Test Accuracy of  frog: 77% (779/1000)
    Test Accuracy of horse: 65% (657/1000)
    Test Accuracy of  ship: 75% (752/1000)
    Test Accuracy of truck: 66% (665/1000)
    Test Accuracy (Overall): 63% (6351/10000)
 %% Cell type:markdown id:944991a2 tags:
 Build a new network with the following structure.
 - It has 3 convolutional layers of kernel size 3 and padding of 1.
 - The first convolutional layer must output 16 channels, the second 32 and the third 64.
 - At each convolutional layer output, we apply a ReLU activation then a MaxPool with kernel size of 2.
 - Then, three fully connected layers, the first two being followed by a ReLU activation and a dropout whose value you will suggest.
 - The first fully connected layer will have an output size of 512.
 - The second fully connected layer will have an output size of 64.
 Compare the results obtained with this new network to those obtained previously.
 %% Cell type:code id: tags:
 ``` python
 import numpy as np
 from torchvision import datasets, transforms
 from torch.utils.data.sampler import SubsetRandomSampler
 # number of subprocesses to use for data loading
 num_workers = 0
 # how many samples per batch to load
 batch_size = 20
 # percentage of training set to use as validation
 valid_size = 0.2
 # convert data to a normalized torch.FloatTensor
 transform = transforms.Compose(
    [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
 )
 # choose the training and test datasets
 train_data = datasets.CIFAR10("data", train=True, download=True, transform=transform)
 test_data = datasets.CIFAR10("data", train=False, download=True, transform=transform)
 # obtain training indices that will be used for validation
 num_train = len(train_data)
 indices = list(range(num_train))
 np.random.shuffle(indices)
 split = int(np.floor(valid_size * num_train))
 train_idx, valid_idx = indices[split:], indices[:split]
 # define samplers for obtaining training and validation batches
 train_sampler = SubsetRandomSampler(train_idx)
 valid_sampler = SubsetRandomSampler(valid_idx)
 # prepare data loaders (combine dataset and sampler)
 train_loader = torch.utils.data.DataLoader(
    train_data, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers
 )
 valid_loader = torch.utils.data.DataLoader(
    train_data, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers
 )
 test_loader = torch.utils.data.DataLoader(
    test_data, batch_size=batch_size, num_workers=num_workers
 )
 # specify the image classes
 classes = [
    "airplane",
    "automobile",
    "bird",
    "cat",
    "deer",
    "dog",
    "frog",
    "horse",
    "ship",
    "truck",
 ]
 ```
 %% Output
    Files already downloaded and verified
    Files already downloaded and verified
 %% Cell type:code id: tags:
 ``` python
 import torch
 import torch.nn as nn
 class Net2(nn.Module):
    def __init__(self, dropout_rate=0.5, num_classes=10):
        super(Net2, self).__init__()
        # Convolutional layers
        self.conv1 = nn.Conv2d(3, 16, 3, 1)
        self.relu1 = nn.ReLU()
        self.pool1 = nn.MaxPool2d(2)
        self.conv2 = nn.Conv2d(16, 32, 3, 1)
        self.relu2 = nn.ReLU()
        self.pool2 = nn.MaxPool2d(2)
        self.conv3 = nn.Conv2d(32, 64, 3, 1)
        self.relu3 = nn.ReLU()
        self.pool3 = nn.MaxPool2d(2)
        # Adaptive pooling to dynamically adjust to input size
        self.adaptive_pool = nn.AdaptiveAvgPool2d((1, 1))
        # Fully connected layers
        self.fc1 = nn.Linear(64, 512)
        self.relu4 = nn.ReLU()
        self.dropout1 = nn.Dropout(dropout_rate)
        self.fc2 = nn.Linear(512, 64)
        self.relu5 = nn.ReLU()
        self.dropout2 = nn.Dropout(dropout_rate)
        self.fc3 = nn.Linear(64, num_classes)
    def forward(self, x):
        # Convolutional layers
        x = self.pool1(self.relu1(self.conv1(x)))
        x = self.pool2(self.relu2(self.conv2(x)))
        x = self.pool3(self.relu3(self.conv3(x)))
        # Adaptive pooling to dynamically adjust to input size
        x = self.adaptive_pool(x)
        # Flatten before fully connected layers
        x = x.view(x.size(0), -1)
        # Fully connected layers
        x = self.dropout1(self.relu4(self.fc1(x)))
        x = self.dropout2(self.relu5(self.fc2(x)))
        x = self.fc3(x)
        return x
 # Instantiate the model
 model2 = Net2()
 # Print the model architecture
 print(model2)
 ```
 %% Output
    Net2(
      (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1))
      (relu1): ReLU()
      (pool1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (conv2): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))
      (relu2): ReLU()
      (pool2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (conv3): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1))
      (relu3): ReLU()
      (pool3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (adaptive_pool): AdaptiveAvgPool2d(output_size=(1, 1))
      (fc1): Linear(in_features=64, out_features=512, bias=True)
      (relu4): ReLU()
      (dropout1): Dropout(p=0.5, inplace=False)
      (fc2): Linear(in_features=512, out_features=64, bias=True)
      (relu5): ReLU()
      (dropout2): Dropout(p=0.5, inplace=False)
      (fc3): Linear(in_features=64, out_features=10, bias=True)
    )
 %% Cell type:code id: tags:
 ``` python
 import torch.optim as optim
 criterion = nn.CrossEntropyLoss()  # specify loss function
 optimizer = optim.SGD(model2.parameters(), lr=0.01)  # specify optimizer
 n_epochs = 30 # number of epochs to train the model
 train_loss_list2 = [] # list to store loss to visualize
 Valid_loss_list2 = []
 valid_loss_min = np.Inf  # track change in validation loss
 for epoch in range(n_epochs):
    # Keep track of training and validation loss
    train_loss = 0.0
    valid_loss = 0.0
    # Train the model
    model2.train()
    for data, target in train_loader:
        # Move tensors to GPU if CUDA is available
        if train_on_gpu:
            data, target = data.cuda(), target.cuda()
        # Clear the gradients of all optimized variables
        optimizer.zero_grad()
        # Forward pass: compute predicted outputs by passing inputs to the model
        output = model2(data)
        # Calculate the batch loss
        loss = criterion(output, target)
        # Backward pass: compute gradient of the loss with respect to model parameters
        loss.backward()
        # Perform a single optimization step (parameter update)
        optimizer.step()
        # Update training loss
        train_loss += loss.item() * data.size(0)
    # Validate the model
    model2.eval()
    for data, target in valid_loader:
        # Move tensors to GPU if CUDA is available
        if train_on_gpu:
            data, target = data.cuda(), target.cuda()
        # Forward pass: compute predicted outputs by passing inputs to the model
        output = model2(data)
        # Calculate the batch loss
        loss = criterion(output, target)
        # Update average validation loss
        valid_loss += loss.item() * data.size(0)
    # Calculate average losses
    train_loss = train_loss / len(train_loader)
    valid_loss = valid_loss / len(valid_loader)
    train_loss_list2.append(train_loss)
    Valid_loss_list2.append(valid_loss)
    # Print training/validation statistics
    print(
        "Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}".format(
            epoch, train_loss, valid_loss
        )
    )
    # Save model if validation loss has decreased
    if valid_loss <= valid_loss_min:
        print(
            "Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...".format(
                valid_loss_min, valid_loss
            )
        )
        torch.save(model2.state_dict(), "model_cifar.pt")
        valid_loss_min = valid_loss
 ```
 %% Output
    Epoch: 0 	Training Loss: 45.993847 	Validation Loss: 45.659072
    Validation loss decreased (inf --> 45.659072).  Saving model ...
    Epoch: 1 	Training Loss: 42.969891 	Validation Loss: 39.458595
    Validation loss decreased (45.659072 --> 39.458595).  Saving model ...
    Epoch: 2 	Training Loss: 38.748317 	Validation Loss: 36.780595
    Validation loss decreased (39.458595 --> 36.780595).  Saving model ...
    Epoch: 3 	Training Loss: 36.406563 	Validation Loss: 34.876807
    Validation loss decreased (36.780595 --> 34.876807).  Saving model ...
    Epoch: 4 	Training Loss: 34.736435 	Validation Loss: 32.692011
    Validation loss decreased (34.876807 --> 32.692011).  Saving model ...
    Epoch: 5 	Training Loss: 33.093783 	Validation Loss: 32.273968
    Validation loss decreased (32.692011 --> 32.273968).  Saving model ...
    Epoch: 6 	Training Loss: 32.013036 	Validation Loss: 29.922774
    Validation loss decreased (32.273968 --> 29.922774).  Saving model ...
    Epoch: 7 	Training Loss: 31.049389 	Validation Loss: 28.974758
    Validation loss decreased (29.922774 --> 28.974758).  Saving model ...
    Epoch: 8 	Training Loss: 30.163901 	Validation Loss: 28.224655
    Validation loss decreased (28.974758 --> 28.224655).  Saving model ...
    Epoch: 9 	Training Loss: 29.254000 	Validation Loss: 27.222523
    Validation loss decreased (28.224655 --> 27.222523).  Saving model ...
    Epoch: 10 	Training Loss: 28.108061 	Validation Loss: 26.104135
    Validation loss decreased (27.222523 --> 26.104135).  Saving model ...
    Epoch: 11 	Training Loss: 27.228062 	Validation Loss: 25.670827
    Validation loss decreased (26.104135 --> 25.670827).  Saving model ...
    Epoch: 12 	Training Loss: 26.315241 	Validation Loss: 24.450983
    Validation loss decreased (25.670827 --> 24.450983).  Saving model ...
    Epoch: 13 	Training Loss: 25.466175 	Validation Loss: 23.767074
    Validation loss decreased (24.450983 --> 23.767074).  Saving model ...
    Epoch: 14 	Training Loss: 24.800794 	Validation Loss: 23.662658
    Validation loss decreased (23.767074 --> 23.662658).  Saving model ...
    Epoch: 15 	Training Loss: 24.069690 	Validation Loss: 22.466158
    Validation loss decreased (23.662658 --> 22.466158).  Saving model ...
    Epoch: 16 	Training Loss: 23.422421 	Validation Loss: 21.784056
    Validation loss decreased (22.466158 --> 21.784056).  Saving model ...
    Epoch: 17 	Training Loss: 22.939551 	Validation Loss: 21.506103
    Validation loss decreased (21.784056 --> 21.506103).  Saving model ...
    Epoch: 18 	Training Loss: 22.376958 	Validation Loss: 21.859394
    Epoch: 19 	Training Loss: 21.936516 	Validation Loss: 20.984741
    Validation loss decreased (21.506103 --> 20.984741).  Saving model ...
    Epoch: 20 	Training Loss: 21.373166 	Validation Loss: 20.880130
    Validation loss decreased (20.984741 --> 20.880130).  Saving model ...
    Epoch: 21 	Training Loss: 20.836367 	Validation Loss: 20.546615
    Validation loss decreased (20.880130 --> 20.546615).  Saving model ...
    Epoch: 22 	Training Loss: 20.522886 	Validation Loss: 20.174353
    Validation loss decreased (20.546615 --> 20.174353).  Saving model ...
    Epoch: 23 	Training Loss: 20.074364 	Validation Loss: 19.755045
    Validation loss decreased (20.174353 --> 19.755045).  Saving model ...
    Epoch: 24 	Training Loss: 19.662466 	Validation Loss: 20.041506
    Epoch: 25 	Training Loss: 19.304117 	Validation Loss: 19.140105
    Validation loss decreased (19.755045 --> 19.140105).  Saving model ...
    Epoch: 26 	Training Loss: 19.006838 	Validation Loss: 19.709049
    Epoch: 27 	Training Loss: 18.619268 	Validation Loss: 19.128607
    Validation loss decreased (19.140105 --> 19.128607).  Saving model ...
    Epoch: 28 	Training Loss: 18.314379 	Validation Loss: 18.985464
    Validation loss decreased (19.128607 --> 18.985464).  Saving model ...
    Epoch: 29 	Training Loss: 18.055548 	Validation Loss: 19.365814
 %% Cell type:code id: tags:
 ``` python
 import matplotlib.pyplot as plt
 plt.plot(range(n_epochs), train_loss_list2, label='train_loss')
 plt.plot(range(n_epochs), Valid_loss_list2, label='Valid_loss')
 plt.xlabel("Epoch")
 plt.ylabel("Loss")
 plt.title("Performance du model")
 plt.legend()
 plt.show()
 ```
 %% Output
 %% Cell type:markdown id: tags:
 Le nouveau modèle est beaucoup plus performant, on a pas de problème d'overfitting contrairement à l'encien modèle.
 %% Cell type:code id: tags:
 ``` python
 # Classification model2 Test
 import torch
 from torchvision import transforms
 from PIL import Image
 # Define the transformation for the input image
 prediction_transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
 ])
 # Load the trained model
 model2 = Net2()
 model2.load_state_dict(torch.load("model_cifar.pt"))
 model2.eval()
 # Function to predict the class of an input image
 def predict_class(image_path):
    # Open the image using PIL
    image = Image.open(image_path)
    # Apply the transformation
    input_image = prediction_transform(image)
    # Add an extra batch dimension
    input_image = input_image.unsqueeze(0)
    # Move the input tensor to the GPU if available
    if torch.cuda.is_available():
        input_image = input_image.cuda()
    # Get the model prediction
    with torch.no_grad():
        output = model2(input_image)
    # Convert the output probabilities to predicted class
    _, predicted_class = torch.max(output, 1)
    return predicted_class.item()
 # Example usage
 image_path = r"C:\Users\ZINEB\Documents\GitHub\mod_4_6-td2\dog.png"
 predicted_class = predict_class(image_path)
 print("Predicted Class:", classes[predicted_class])
 ```
 %% Output
    Predicted Class: cat
 %% Cell type:markdown id:bc381cf4 tags:
 ## Exercise 2: Quantization: try to compress the CNN to save space
 Quantization doc is available from https://pytorch.org/docs/stable/quantization.html#torch.quantization.quantize_dynamic
 The Exercise is to quantize post training the above CNN model. Compare the size reduction and the impact on the classification accuracy
 The size of the model is simply the size of the file.
 %% Cell type:code id:ef623c26 tags:
 ``` python
 import os
 def print_size_of_model(model2, label=""):
    torch.save(model2.state_dict(), "temp.p")
    size = os.path.getsize("temp.p")
    print("model: ", label, " \t", "Size (KB):", size / 1e3)
    os.remove("temp.p")
    return size
 print_size_of_model(model2, "fp32")
 ```
 %% Output
    model:  fp32  	 Size (KB): 365.058
    365058
 %% Cell type:markdown id:05c4e9ad tags:
 Post training quantization example
 %% Cell type:code id:c4c65d4b tags:
 ``` python
 import torch.quantization
 quantized_model2 = torch.quantization.quantize_dynamic(model2, dtype=torch.qint8)
 print_size_of_model(quantized_model2, "int8")
 ```
 %% Output
    model:  int8  	 Size (KB): 168.478
    168478
 %% Cell type:markdown id:7b108e17 tags:
 For each class, compare the classification test accuracy of the initial model and the quantized model. Also give the overall test accuracy for both models.
 %% Cell type:code id: tags:
 ``` python
 import torch
 import torch.nn as nn
 from torchvision import datasets, transforms
 from torch.utils.data.sampler import SubsetRandomSampler
 import os
 import numpy as np
 ```
 %% Cell type:markdown id:a0a34b90 tags:
 Try training aware quantization to mitigate the impact on the accuracy (doc available here https://pytorch.org/docs/stable/quantization.html#torch.quantization.quantize_dynamic)
 %% Cell type:code id: tags:
 ``` python
 # function to evaluate the accuracy of the model
 def evaluate_model(model, dataloader):
    model.eval()
    correct = {cls: 0 for cls in classes}
    total = {cls: 0 for cls in classes}
    with torch.no_grad():
        for data in dataloader:
            images, labels = data
            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            for i in range(len(labels)):
                total[classes[labels[i]]] += 1
                correct[classes[labels[i]]] += int(predicted[i] == labels[i])
    accuracy = {cls: correct[cls] / total[cls] for cls in classes}
    return accuracy
 ```
 %% Cell type:code id: tags:
 ``` python
 # Evaluate the accuracy of the original model on the test set
 accuracy_before_quantization = evaluate_model(model2, test_loader)
 overall_accuracy_before_quantization = sum(accuracy_before_quantization.values()) / len(classes)
 print("Accuracy before quantization:")
 for cls, acc in accuracy_before_quantization.items():
    print(f"{cls}: {acc:.4f}")
 print(f"Overall Accuracy: {overall_accuracy_before_quantization:.4f}")
 # Post-training quantization
 quantized_model = torch.quantization.quantize_dynamic(model2, dtype=torch.qint8)
 # Print the size of the quantized model
 print_size_of_model(quantized_model, "int8")
 # Evaluate the accuracy of the quantized model on the test set
 accuracy_after_quantization = evaluate_model(quantized_model, test_loader)
 overall_accuracy_after_quantization = sum(accuracy_after_quantization.values()) / len(classes)
 print("Accuracy after quantization:")
 for cls, acc in accuracy_after_quantization.items():
    print(f"{cls}: {acc:.4f}")
 print(f"Overall Accuracy: {overall_accuracy_after_quantization:.4f}")
 ```
 %% Output
    Accuracy before quantization:
    airplane: 0.6300
    automobile: 0.8050
    bird: 0.3740
    cat: 0.3750
    deer: 0.7130
    dog: 0.5410
    frog: 0.7490
    horse: 0.7570
    ship: 0.8580
    truck: 0.8130
    Overall Accuracy: 0.6615
    model:  int8  	 Size (KB): 168.478
    Accuracy after quantization:
    airplane: 0.6270
    automobile: 0.8040
    bird: 0.3770
    cat: 0.3790
    deer: 0.7100
    dog: 0.5370
    frog: 0.7490
    horse: 0.7560
    ship: 0.8580
    truck: 0.8140
    Overall Accuracy: 0.6611
 %% Cell type:markdown id:201470f9 tags:
 ## Exercise 3: working with pre-trained models.
 PyTorch offers several pre-trained models https://pytorch.org/vision/0.8/models.html
 We will use ResNet50 trained on ImageNet dataset (https://www.image-net.org/index.php). Use the following code with the files `imagenet-simple-labels.json` that contains the imagenet labels and the image dog.png that we will use as test.
 %% Cell type:code id:b4d13080 tags:
 ``` python
 import json
 from PIL import Image
 # Choose an image to pass through the model
 test_image = "dog.png"
 # Configure matplotlib for pretty inline plots
 #%matplotlib inline
 #%config InlineBackend.figure_format = 'retina'
 # Prepare the labels
 with open("imagenet-simple-labels.json") as f:
    labels = json.load(f)
 # First prepare the transformations: resize the image to what the model was trained on and convert it to a tensor
 data_transform = transforms.Compose(
    [
        transforms.Resize((224, 224)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
    ]
 )
 # Load the image
 image = Image.open(test_image)
 plt.imshow(image), plt.xticks([]), plt.yticks([])
 # Now apply the transformation, expand the batch dimension, and send the image to the GPU
 # image = data_transform(image).unsqueeze(0).cuda()
 image = data_transform(image).unsqueeze(0)
 # Download the model if it's not there already. It will take a bit on the first run, after that it's fast
 model = models.resnet50(pretrained=True)
 # Send the model to the GPU
 # model.cuda()
 # Set layers such as dropout and batchnorm in evaluation mode
 model.eval()
 # Get the 1000-dimensional model output
 out = model(image)
 # Find the predicted class
 print("Predicted class is: {}".format(labels[out.argmax()]))
 ```
 %% Output
    c:\Users\ZINEB\anaconda3\lib\site-packages\torchvision\models\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
      warnings.warn(
    c:\Users\ZINEB\anaconda3\lib\site-packages\torchvision\models\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet50_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet50_Weights.DEFAULT` to get the most up-to-date weights.
      warnings.warn(msg)
    Downloading: "https://download.pytorch.org/models/resnet50-0676ba61.pth" to C:\Users\ZINEB/.cache\torch\hub\checkpoints\resnet50-0676ba61.pth
    100%|██████████| 97.8M/97.8M [00:17<00:00, 5.92MB/s]
    Predicted class is: Golden Retriever
 %% Cell type:markdown id:184cfceb tags:
 Experiments:
 Study the code and the results obtained. Possibly add other images downloaded from the internet.
 What is the size of the model? Quantize it and then check if the model is still able to correctly classify the other images.
 Experiment with other pre-trained CNN models.
 %% Cell type:code id: tags:
 ``` python
 import json
 from PIL import Image
 import matplotlib.pyplot as plt
 from torchvision import transforms, models
 # Choose an image to pass through the model
 test_image = "cat.png"
 # Prepare the labels
 with open("imagenet-simple-labels.json") as f:
    labels = json.load(f)
 # Prepare the transformations: resize the image to what the model was trained on and convert it to a tensor
 data_transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    # Ensure the normalization parameters match the number of channels (3 for RGB)
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
 ])
 # Load the image
 image = Image.open(test_image)
 # Convert the image to RGB if it's not in that mode
 if image.mode != 'RGB':
    image = image.convert('RGB')
 # Apply the transformation, expand the batch dimension, and convert to float
 image_tensor = data_transform(image).unsqueeze(0).float()
 # Download the model if it's not there already. It will take a bit on the first run, after that it's fast
 model = models.resnet50(pretrained=True)
 # Set the model to evaluation mode
 model.eval()
 # Get the 1000-dimensional model output
 out = model(image_tensor)
 # Find the predicted class
 predicted_class = labels[out.argmax()]
 # Display the image
 plt.imshow(image)
 plt.title(f"Predicted class: {predicted_class}")
 plt.axis("off")
 plt.show()
 ```
 %% Output
 %% Cell type:code id: tags:
 ``` python
 import json
 from PIL import Image
 import matplotlib.pyplot as plt
 from torchvision import transforms, models
 # Choose an image to pass through the model
 test_image = "frog.png"
 # Prepare the labels
 with open("imagenet-simple-labels.json") as f:
    labels = json.load(f)
 # Prepare the transformations: resize the image to what the model was trained on and convert it to a tensor
 data_transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    # Ensure the normalization parameters match the number of channels (3 for RGB)
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
 ])
 # Load the image
 image = Image.open(test_image)
 # Convert the image to RGB if it's not in that mode
 if image.mode != 'RGB':
    image = image.convert('RGB')
 # Apply the transformation, expand the batch dimension, and convert to float
 image_tensor = data_transform(image).unsqueeze(0).float()
 # Download the model if it's not there already. It will take a bit on the first run, after that it's fast
 model = models.resnet50(pretrained=True)
 # Set the model to evaluation mode
 model.eval()
 # Get the 1000-dimensional model output
 out = model(image_tensor)
 # Find the predicted class
 predicted_class = labels[out.argmax()]
 # Display the image
 plt.imshow(image)
 plt.title(f"Predicted class: {predicted_class}")
 plt.axis("off")
 plt.show()
 ```
 %% Output
 %% Cell type:code id: tags:
 ``` python
 #size of the model
 total_para = sum(p.numel() for p in model.parameters())
 print(f"Number of parameters: {total_para}")
 ```
 %% Output
    Number of parameters: 25557032
 %% Cell type:code id: tags:
 ``` python
 #Quantize the Model
 from torch.quantization import quantize_dynamic
 test_image = "dog.png"
 # Prepare the labels
 with open("imagenet-simple-labels.json") as f:
    labels = json.load(f)
 # Prepare the transformations: resize the image to what the model was trained on and convert it to a tensor
 data_transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
 ])
 # Load the image
 image = Image.open(test_image)
 plt.imshow(image), plt.xticks([]), plt.yticks([])
 # Apply the transformation, expand the batch dimension
 image = data_transform(image).unsqueeze(0)
 # Load the pretrained ResNet50 model
 model = models.resnet50(pretrained=True)
 model.eval()
 # Get the 1000-dimensional model output
 out = model(image)
 predicted_class = labels[out.argmax()]
 print("Predicted class is: {}".format(predicted_class))
 # Quantize the model
 quantized_model = quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
 quantized_model.eval()
 out_quantized = quantized_model(image)
 predicted_class_quantized = labels[out_quantized.argmax()]
 print("Quantized model predicted class is: {}".format(predicted_class_quantized))
 ```
 %% Output
    Predicted class is: Golden Retriever
    Quantized model predicted class is: Golden Retriever
 %% Cell type:code id: tags:
 ``` python
 import json
 from PIL import Image
 import matplotlib.pyplot as plt
 from torchvision import transforms, models
 import torch
 # Choose an image to pass through the model
 test_image = "frog.png"
 # Prepare the labels
 with open("imagenet-simple-labels.json") as f:
    labels = json.load(f)
 # Prepare the transformations: resize the image to what the model was trained on and convert it to a tensor
 data_transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
 ])
 # Load the image
 image = Image.open(test_image)
 # Convert the image to RGB if it's not in that mode
 if image.mode != 'RGB':
    image = image.convert('RGB')
 # Apply the transformation, expand the batch dimension, and convert to float
 image_tensor = data_transform(image).unsqueeze(0).float()
 # Download the model if it's not there already. It will take a bit on the first run, after that it's fast
 model = models.resnet50(pretrained=True)
 # Set the model to evaluation mode
 model.eval()
 # Get the 1000-dimensional model output
 out = model(image_tensor)
 # Find the predicted class
 predicted_class = labels[out.argmax()]
 # Display the image and predicted class
 plt.imshow(image)
 plt.title(f"Predicted class: {predicted_class}")
 plt.axis("off")
 plt.show()
 # Quantize the model dynamically
 quantized_model = torch.quantization.quantize_dynamic(
    model, {torch.nn.Conv2d, torch.nn.Linear}, dtype=torch.qint8
 )
 quantized_model.eval()
 # Get the 1000-dimensional model output for the quantized model
 out_quantized = quantized_model(image_tensor)
 # Find the predicted class for the quantized model
 predicted_class_quantized = labels[out_quantized.argmax()]
 # Display the quantized model's predicted class
 print(f"Quantized model predicted class: {predicted_class_quantized}")
 ```
 %% Output
    Quantized model predicted class: tailed frog
 %% Cell type:code id: tags:
 ``` python
 #AlexNet Pre-trained model
 test_image = "dog.png"
 # Prepare the labels
 with open("imagenet-simple-labels.json") as f:
    labels = json.load(f)
 # Prepare the transformations
 data_transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
 ])
 # Load the image
 image = Image.open(test_image)
 plt.imshow(image)
 plt.xticks([]), plt.yticks([])
 plt.show()
 image = data_transform(image).unsqueeze(0)
 model = models.alexnet(pretrained=True)
 # Set layers such as dropout and batchnorm in evaluation mode
 model.eval()
 # Get the 1000-dimensional model output
 out = model(image)
 # Find the predicted class
 print("Predicted class is: {}".format(labels[out.argmax()]))
 ```
 %% Output
    c:\Users\ZINEB\anaconda3\lib\site-packages\torchvision\models\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=AlexNet_Weights.IMAGENET1K_V1`. You can also use `weights=AlexNet_Weights.DEFAULT` to get the most up-to-date weights.
      warnings.warn(msg)
    Downloading: "https://download.pytorch.org/models/alexnet-owt-7be5be79.pth" to C:\Users\ZINEB/.cache\torch\hub\checkpoints\alexnet-owt-7be5be79.pth
    100%|██████████| 233M/233M [00:42<00:00, 5.71MB/s]
    Predicted class is: Golden Retriever
 %% Cell type:code id: tags:
 ``` python
 #VGG trained-model
 # Choose an image to pass through the model
 test_image = "dog.png"
 # Prepare the labels
 with open("imagenet-simple-labels.json") as f:
    labels = json.load(f)
 # Prepare the transformations: resize the image to what the model was trained on and convert it to a tensor
 data_transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
 ])
 # Load the image
 image = Image.open(test_image)
 plt.imshow(image)
 plt.xticks([]), plt.yticks([])
 plt.show()
 image = data_transform(image).unsqueeze(0)
 model = models.vgg16(pretrained=True)
 model.eval()
 # Get the 1000-dimensional model output
 out = model(image)
 # Find the predicted class
 print("Predicted class is: {}".format(labels[out.argmax()]))
 ```
 %% Output
    c:\Users\ZINEB\anaconda3\lib\site-packages\torchvision\models\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
      warnings.warn(msg)
    Downloading: "https://download.pytorch.org/models/vgg16-397923af.pth" to C:\Users\ZINEB/.cache\torch\hub\checkpoints\vgg16-397923af.pth
    100%|██████████| 528M/528M [01:46<00:00, 5.21MB/s]
    Predicted class is: Golden Retriever
 %% Cell type:markdown id:5d57da4b tags:
 ## Exercise 4: Transfer Learning
 For this work, we will use a pre-trained model (ResNet18) as a descriptor extractor and will refine the classification by training only the last fully connected layer of the network. Thus, the output layer of the pre-trained network will be replaced by a layer adapted to the new classes to be recognized which will be in our case ants and bees.
 Download and unzip in your working directory the dataset available at the address :
 https://download.pytorch.org/tutorial/hymenoptera_data.zip
 Execute the following code in order to display some images of the dataset.
 %% Cell type:code id:be2d31f5 tags:
 ``` python
 import os
 import matplotlib.pyplot as plt
 import numpy as np
 import torch
 import torchvision
 from torchvision import datasets, transforms
 # Data augmentation and normalization for training
 # Just normalization for validation
 data_transforms = {
    "train": transforms.Compose(
        [
            transforms.RandomResizedCrop(
                224
            ),  # ImageNet models were trained on 224x224 images
            transforms.RandomHorizontalFlip(),  # flip horizontally 50% of the time - increases train set variability
            transforms.ToTensor(),  # convert it to a PyTorch tensor
            transforms.Normalize(
                [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
            ),  # ImageNet models expect this norm
        ]
    ),
    "val": transforms.Compose(
        [
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
        ]
    ),
 }
 data_dir = "hymenoptera_data"
 # Create train and validation datasets and loaders
 image_datasets = {
    x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x])
    for x in ["train", "val"]
 }
 dataloaders = {
    x: torch.utils.data.DataLoader(
        image_datasets[x], batch_size=4, shuffle=True, num_workers=0
    )
    for x in ["train", "val"]
 }
 dataset_sizes = {x: len(image_datasets[x]) for x in ["train", "val"]}
 class_names = image_datasets["train"].classes
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 # Helper function for displaying images
 def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    # Un-normalize the images
    inp = std * inp + mean
    # Clip just in case
    inp = np.clip(inp, 0, 1)
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)  # pause a bit so that plots are updated
    plt.show()
 # Get a batch of training data
 inputs, classes = next(iter(dataloaders["train"]))
 # Make a grid from batch
 out = torchvision.utils.make_grid(inputs)
 imshow(out, title=[class_names[x] for x in classes])
 ```
 %% Output
 %% Cell type:markdown id:bbd48800 tags:
 Now, execute the following code which uses a pre-trained model ResNet18 having replaced the output layer for the ants/bees classification and performs the model training by only changing the weights of this output layer.
 %% Cell type:code id:572d824c tags:
 ``` python
 import copy
 import os
 import time
 import matplotlib.pyplot as plt
 import numpy as np
 import torch
 import torch.nn as nn
 import torch.optim as optim
 import torchvision
 from torch.optim import lr_scheduler
 from torchvision import datasets, transforms
 # Data augmentation and normalization for training
 # Just normalization for validation
 data_transforms = {
    "train": transforms.Compose(
        [
            transforms.RandomResizedCrop(
                224
            ),  # ImageNet models were trained on 224x224 images
            transforms.RandomHorizontalFlip(),  # flip horizontally 50% of the time - increases train set variability
            transforms.ToTensor(),  # convert it to a PyTorch tensor
            transforms.Normalize(
                [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
            ),  # ImageNet models expect this norm
        ]
    ),
    "val": transforms.Compose(
        [
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
        ]
    ),
 }
 data_dir = "hymenoptera_data"
 # Create train and validation datasets and loaders
 image_datasets = {
    x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x])
    for x in ["train", "val"]
 }
 dataloaders = {
    x: torch.utils.data.DataLoader(
        image_datasets[x], batch_size=4, shuffle=True, num_workers=4
    )
    for x in ["train", "val"]
 }
 dataset_sizes = {x: len(image_datasets[x]) for x in ["train", "val"]}
 class_names = image_datasets["train"].classes
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 # Helper function for displaying images
 def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    # Un-normalize the images
    inp = std * inp + mean
    # Clip just in case
    inp = np.clip(inp, 0, 1)
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)  # pause a bit so that plots are updated
    plt.show()
 # Get a batch of training data
 # inputs, classes = next(iter(dataloaders['train']))
 # Make a grid from batch
 # out = torchvision.utils.make_grid(inputs)
 # imshow(out, title=[class_names[x] for x in classes])
 # training
 def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time()
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    epoch_time = []  # we'll keep track of the time needed for each epoch
    for epoch in range(num_epochs):
        epoch_start = time.time()
        print("Epoch {}/{}".format(epoch + 1, num_epochs))
        print("-" * 10)
        # Each epoch has a training and validation phase
        for phase in ["train", "val"]:
            if phase == "train":
                scheduler.step()
                model.train()  # Set model to training mode
            else:
                model.eval()  # Set model to evaluate mode
            running_loss = 0.0
            running_corrects = 0
            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)
                # zero the parameter gradients
                optimizer.zero_grad()
                # Forward
                # Track history if only in training phase
                with torch.set_grad_enabled(phase == "train"):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                    # backward + optimize only if in training phase
                    if phase == "train":
                        loss.backward()
                        optimizer.step()
                # Statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            print("{} Loss: {:.4f} Acc: {:.4f}".format(phase, epoch_loss, epoch_acc))
            # Deep copy the model
            if phase == "val" and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
        # Add the epoch time
        t_epoch = time.time() - epoch_start
        epoch_time.append(t_epoch)
        print()
    time_elapsed = time.time() - since
    print(
        "Training complete in {:.0f}m {:.0f}s".format(
            time_elapsed // 60, time_elapsed % 60
        )
    )
    print("Best val Acc: {:4f}".format(best_acc))
    # Load best model weights
    model.load_state_dict(best_model_wts)
    return model, epoch_time
 # Download a pre-trained ResNet18 model and freeze its weights
 model = torchvision.models.resnet18(pretrained=True)
 for param in model.parameters():
    param.requires_grad = False
 # Replace the final fully connected layer
 # Parameters of newly constructed modules have requires_grad=True by default
 num_ftrs = model.fc.in_features
 model.fc = nn.Linear(num_ftrs, 2)
 # Send the model to the GPU
 model = model.to(device)
 # Set the loss function
 criterion = nn.CrossEntropyLoss()
 # Observe that only the parameters of the final layer are being optimized
 optimizer_conv = optim.SGD(model.fc.parameters(), lr=0.001, momentum=0.9)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer_conv, step_size=7, gamma=0.1)
 model, epoch_time = train_model(
    model, criterion, optimizer_conv, exp_lr_scheduler, num_epochs=10
 )
 ```
+%% Output
+    c:\Users\ZINEB\anaconda3\lib\site-packages\torchvision\models\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.
+      warnings.warn(msg)
+    Downloading: "https://download.pytorch.org/models/resnet18-f37072fd.pth" to C:\Users\ZINEB/.cache\torch\hub\checkpoints\resnet18-f37072fd.pth
+    100%|██████████| 44.7M/44.7M [00:10<00:00, 4.58MB/s]
+    Epoch 1/10
+    ----------
+    c:\Users\ZINEB\anaconda3\lib\site-packages\torch\optim\lr_scheduler.py:136: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
+      warnings.warn("Detected call of `lr_scheduler.step()` before `optimizer.step()`. "
+    train Loss: 0.6227 Acc: 0.6680
+    val Loss: 0.3421 Acc: 0.8758
+    Epoch 2/10
+    ----------
+    train Loss: 0.7198 Acc: 0.7090
+    val Loss: 0.1835 Acc: 0.9477
+    Epoch 3/10
+    ----------
+    train Loss: 0.5178 Acc: 0.7746
+    val Loss: 0.1933 Acc: 0.9412
+    Epoch 4/10
+    ----------
+    train Loss: 0.4848 Acc: 0.8074
+    val Loss: 0.2029 Acc: 0.9281
+    Epoch 5/10
+    ----------
+    train Loss: 0.5139 Acc: 0.8074
+    val Loss: 0.2291 Acc: 0.9150
+    Epoch 6/10
+    ----------
+    train Loss: 0.5792 Acc: 0.8033
+    val Loss: 0.1806 Acc: 0.9412
+    Epoch 7/10
+    ----------
+    train Loss: 0.3262 Acc: 0.8607
+    val Loss: 0.2058 Acc: 0.9412
+    Epoch 8/10
+    ----------
+    train Loss: 0.3552 Acc: 0.8648
+    val Loss: 0.1882 Acc: 0.9412
+    Epoch 9/10
+    ----------
+    train Loss: 0.3464 Acc: 0.8484
+    val Loss: 0.1844 Acc: 0.9477
+    Epoch 10/10
+    ----------
+    train Loss: 0.3932 Acc: 0.8074
+    val Loss: 0.1730 Acc: 0.9412
+    Training complete in 8m 30s
+    Best val Acc: 0.947712
 %% Cell type:markdown id:bbd48800 tags:
 Experiments:
 Study the code and the results obtained.
 Modify the code and add an "eval_model" function to allow
 the evaluation of the model on a test set (different from the learning and validation sets used during the learning phase). Study the results obtained.
 Now modify the code to replace the current classification layer with a set of two layers using a "relu" activation function for the middle layer, and the "dropout" mechanism for both layers. Renew the experiments and study the results obtained.
 Apply ther quantization (post and quantization aware) and evaluate impact on model size and accuracy.
+%% Cell type:code id: tags:
+``` python
+#eval_model
+def eval_model(model, dataloader, criterion):
+    model.eval()  # Set the model to evaluate mode
+    running_loss = 0.0
+    running_corrects = 0
+    with torch.no_grad():
+        for inputs, labels in dataloader:
+            inputs = inputs.to(device)
+            labels = labels.to(device)
+            outputs = model(inputs)
+            _, preds = torch.max(outputs, 1)
+            loss = criterion(outputs, labels)
+            running_loss += loss.item() * inputs.size(0)
+            running_corrects += torch.sum(preds == labels.data)
+    loss = running_loss / len(dataloader.dataset)
+    accuracy = running_corrects.double() / len(dataloader.dataset)
+    print("Evaluation Loss: {:.4f} Accuracy: {:.4f}".format(loss, accuracy))
+    return loss, accuracy
+```
+%% Cell type:code id: tags:
+``` python
+# Download a pre-trained ResNet18 model and freeze its weights
+model = torchvision.models.resnet18(pretrained=True)
+# Replace the final fully connected layer with two layers and dropout
+num_ftrs = model.fc.in_features
+model.fc = nn.Sequential(
+    nn.Linear(num_ftrs, 256),  # Add a middle layer with 256 units
+    nn.ReLU(),  # ReLU activation for the middle layer
+    nn.Dropout(0.5),  # Dropout with a probability of 0.5
+    nn.Linear(256, 2),  # Output layer with 2 classes
+)
+# Send the model to the GPU
+model = model.to(device)
+# Set the loss function
+criterion = nn.CrossEntropyLoss()
+# Observe that all parameters are being optimized
+optimizer_all = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
+exp_lr_scheduler = lr_scheduler.StepLR(optimizer_all, step_size=7, gamma=0.1)
+# Train the model with the new architecture
+model, epoch_time = train_model(model, criterion, optimizer_all, exp_lr_scheduler, num_epochs=10)
+# Evaluate the model on the test set
+test_dataloader = torch.utils.data.DataLoader(
+    image_datasets["val"], batch_size=4, shuffle=False, num_workers=4
+)
+eval_model(model, test_dataloader, criterion)
+```
+%% Output
+    c:\Users\ZINEB\anaconda3\lib\site-packages\torchvision\models\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
+      warnings.warn(
+    c:\Users\ZINEB\anaconda3\lib\site-packages\torchvision\models\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.
+      warnings.warn(msg)
+    Epoch 1/10
+    ----------
+    c:\Users\ZINEB\anaconda3\lib\site-packages\torch\optim\lr_scheduler.py:136: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
+      warnings.warn("Detected call of `lr_scheduler.step()` before `optimizer.step()`. "
+    train Loss: 0.6425 Acc: 0.6680
+    val Loss: 0.3366 Acc: 0.9346
+    Epoch 2/10
+    ----------
+    train Loss: 0.4859 Acc: 0.7582
+    val Loss: 0.2422 Acc: 0.9412
+    Epoch 3/10
+    ----------
+    train Loss: 0.4214 Acc: 0.8279
+    val Loss: 0.2159 Acc: 0.9346
+    Epoch 4/10
+    ----------
+    train Loss: 0.3997 Acc: 0.8361
+    val Loss: 0.2607 Acc: 0.9085
+    Epoch 5/10
+    ----------
+    train Loss: 0.4037 Acc: 0.8115
+    val Loss: 0.2860 Acc: 0.8824
+    Epoch 6/10
+    ----------
+    train Loss: 0.3597 Acc: 0.8443
+    val Loss: 0.2590 Acc: 0.9085
+    Epoch 7/10
+    ----------
+    train Loss: 0.2971 Acc: 0.8770
+    val Loss: 0.2063 Acc: 0.9281
+    Epoch 8/10
+    ----------
+    train Loss: 0.3260 Acc: 0.8566
+    val Loss: 0.1808 Acc: 0.9412
+    Epoch 9/10
+    ----------
+    train Loss: 0.3064 Acc: 0.8730
+    val Loss: 0.1748 Acc: 0.9477
+    Epoch 10/10
+    ----------
+    train Loss: 0.2438 Acc: 0.9139
+    val Loss: 0.1790 Acc: 0.9477
+    Training complete in 8m 28s
+    Best val Acc: 0.947712
+    Evaluation Loss: 0.1748 Accuracy: 0.9477
+    (0.1747902406309579, tensor(0.9477, dtype=torch.float64))
 %% Cell type:markdown id:04a263f0 tags:
 ## Optional
 Try this at home!!
 Pytorch offers a framework to export a given CNN to your selfphone (either android or iOS). Have a look at the tutorial https://pytorch.org/mobile/home/
 The Exercise consists in deploying the CNN of Exercise 4 in your phone and then test it on live.
 %% Cell type:markdown id:fe954ce4 tags:
 ## Author
 Alberto BOSIO - Ph. D.