From bf566dd77cb5b82cf2902b96109ca5008b76fa28 Mon Sep 17 00:00:00 2001
From: HeberArteagaJ <heberarteagajimenez@gmail.com>
Date: Wed, 4 Dec 2024 17:41:59 +0100
Subject: [PATCH] TD update
---
TD2 Deep Learning.ipynb | 305 ++++++++++++++++++++++------------------
1 file changed, 170 insertions(+), 135 deletions(-)
diff --git a/TD2 Deep Learning.ipynb b/TD2 Deep Learning.ipynb
index cf9b9c8..c9d5504 100644
--- a/TD2 Deep Learning.ipynb
+++ b/TD2 Deep Learning.ipynb
@@ -85,16 +85,6 @@
"%pip install torch torchvision"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b22dbda1",
- "metadata": {},
- "outputs": [],
- "source": [
- "%pip install torch torchvision torchaudio"
- ]
- },
{
"cell_type": "markdown",
"id": "0882a636",
@@ -235,6 +225,16 @@
" print(\"CUDA is available! Training on GPU ...\")"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "5330c0f3",
+ "metadata": {},
+ "source": [
+ "As I am working on a MacBook Pro with ARM processor, I have to check if MPS is available for use with PyTorch.\n",
+ "\n",
+ "DISCLAIMER: Due to this situation, in some parts of the code there were complications since MPS does not have all CUDA implementations, so the CPU had to be used or the code had to be slightly modified."
+ ]
+ },
{
"cell_type": "code",
"execution_count": 3,
@@ -3765,104 +3765,82 @@
},
{
"cell_type": "code",
- "execution_count": 155,
- "id": "a8a0f8cf",
+ "execution_count": 163,
+ "id": "b59e14eb",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
+ "Epoch 1/10\n",
+ "----------\n",
+ "train Loss: 0.6897 Acc: 0.5410\n",
+ "val Loss: 0.6358 Acc: 0.6667\n",
+ "\n",
+ "Epoch 2/10\n",
+ "----------\n",
+ "train Loss: 0.6833 Acc: 0.5369\n",
+ "val Loss: 0.6121 Acc: 0.7974\n",
+ "\n",
+ "Epoch 3/10\n",
+ "----------\n",
+ "train Loss: 0.6257 Acc: 0.6516\n",
+ "val Loss: 0.5742 Acc: 0.8170\n",
+ "\n",
+ "Epoch 4/10\n",
+ "----------\n",
+ "train Loss: 0.6026 Acc: 0.6762\n",
+ "val Loss: 0.5366 Acc: 0.9150\n",
+ "\n",
+ "Epoch 5/10\n",
+ "----------\n",
+ "train Loss: 0.5846 Acc: 0.7213\n",
+ "val Loss: 0.4959 Acc: 0.9346\n",
+ "\n",
+ "Epoch 6/10\n",
+ "----------\n",
+ "train Loss: 0.5483 Acc: 0.7705\n",
+ "val Loss: 0.4574 Acc: 0.9216\n",
+ "\n",
+ "Epoch 7/10\n",
+ "----------\n",
+ "train Loss: 0.4907 Acc: 0.8156\n",
+ "val Loss: 0.4231 Acc: 0.9281\n",
+ "\n",
+ "Epoch 8/10\n",
+ "----------\n",
+ "train Loss: 0.4690 Acc: 0.8238\n",
+ "val Loss: 0.3889 Acc: 0.9477\n",
+ "\n",
+ "Epoch 9/10\n",
+ "----------\n",
+ "train Loss: 0.4716 Acc: 0.8238\n",
+ "val Loss: 0.3541 Acc: 0.9477\n",
+ "\n",
+ "Epoch 10/10\n",
+ "----------\n",
+ "train Loss: 0.4422 Acc: 0.8279\n",
+ "val Loss: 0.3406 Acc: 0.9542\n",
+ "\n",
+ "Training complete in 9m 55s\n",
+ "Best val Acc: 0.954248\n",
"model: fp32 \t Size (KB): 45040.57\n",
- "model: int8 \t Size (KB): 44844.646\n"
+ "Test Loss: 0.3238 Acc: 0.9388\n",
+ "model: int8 \t Size (KB): 45040.57\n"
]
},
{
"data": {
"text/plain": [
- "44844646"
+ "45040570"
]
},
- "execution_count": 155,
+ "execution_count": 163,
"metadata": {},
"output_type": "execute_result"
}
],
- "source": [
- "import torch\n",
- "import torch.quantization\n",
- "\n",
- "# Mueve el modelo a la CPU\n",
- "model = model.to(\"cpu\")\n",
- "\n",
- "# Imprime el tamaño del modelo en FP32\n",
- "print_size_of_model(model, \"fp32\")\n",
- "\n",
- "# Configura el motor de cuantización y realiza la cuantización dinámica\n",
- "torch.backends.quantized.engine = 'qnnpack'\n",
- "quantized_model = torch.quantization.quantize_dynamic(model, dtype=torch.qint8)\n",
- "\n",
- "# Imprime el tamaño del modelo cuantizado en INT8\n",
- "print_size_of_model(quantized_model, \"int8\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "14e5a685",
- "metadata": {},
- "source": [
- "It can be observed that after applying post-training quantization, the model size does not change significantly compared to the original size.\n",
- "\n",
- "This may be because ResNet18 is a relatively small model, with a moderate number of parameters compared to larger architectures. As a result, the reduction in model size is limited.\n",
- "\n",
- "Additionally, post-training quantization is typically applied to layers that have a large number of parameters, such as Linear and Conv2d layers. However, in models like ResNet18, many of the layers are small or have fewer parameters, which can reduce the impact of quantization on the overall model size."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "1891c5df",
- "metadata": {},
- "source": [
- "#### b. Quantization-Aware Training"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "1e6894d1",
- "metadata": {},
- "outputs": [],
- "source": [
- "import torch\n",
- "import torch.nn as nn\n",
- "import torch.optim as optim\n",
- "from torchvision import models\n",
- "\n",
- "# Load ResNet18 model\n",
- "model = models.resnet18(pretrained=True)\n",
- "\n",
- "# Replace the current clasification layer with a set of two layers and Dropout \n",
- "model.fc = nn.Sequential(\n",
- " nn.Dropout(0.4), # Dropout before the first custom layer\n",
- " nn.Linear(model.fc.in_features, 128), # First fully connected layer\n",
- " nn.ReLU(), # Activation function\n",
- " nn.Dropout(0.4), # Dropout after the first custom layer\n",
- " nn.Linear(128, 2) # Output layer for binary classification\n",
- ")\n",
- "\n",
- "model = model.to(device)\n",
- "\n",
- "model = torch.quantization.prepare_qat(model, inplace=False)\n",
- "\n",
- "print(model)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "892f877a",
- "metadata": {},
- "outputs": [],
"source": [
"import copy\n",
"import os\n",
@@ -3877,8 +3855,8 @@
"from torch.optim import lr_scheduler\n",
"from torchvision import datasets, transforms\n",
"\n",
- "\n",
- "# model = model_resnet18_v3\n",
+ "device = torch.device(\"cpu\")\n",
+ "os.environ[\"PYTORCH_ENABLE_MPS_FALLBACK\"] = \"1\"\n",
"\n",
"def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n",
" since = time.time()\n",
@@ -3929,7 +3907,8 @@
" running_corrects += torch.sum(preds == labels.data)\n",
"\n",
" epoch_loss = running_loss / dataset_sizes[phase]\n",
- " epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
+ " # epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
+ " epoch_acc = running_corrects.float() / dataset_sizes[phase]\n",
"\n",
" print(\"{} Loss: {:.4f} Acc: {:.4f}\".format(phase, epoch_loss, epoch_acc))\n",
"\n",
@@ -3955,17 +3934,28 @@
" model.load_state_dict(best_model_wts)\n",
" return model, epoch_time\n",
"\n",
- "# Download a pre-trained ResNet18 model and freeze its weights\n",
+ "\n",
+ "# Replace the current clasification layer with a set of two layers and Dropout \n",
+ "# Load ResNet18 model\n",
"model = torchvision.models.resnet18(pretrained=True)\n",
+ "\n",
+ "# Freeze the earlier layers for fine-tuning\n",
"for param in model.parameters():\n",
" param.requires_grad = False\n",
"\n",
- "# Replace the final fully connected layer\n",
- "# Parameters of newly constructed modules have requires_grad=True by default\n",
"num_ftrs = model.fc.in_features\n",
- "model.fc = nn.Linear(num_ftrs, 2)\n",
- "# Send the model to the GPU\n",
+ "\n",
+ "model.fc = nn.Sequential(\n",
+ " nn.Dropout(0.4), # Dropout before the first custom layer\n",
+ " nn.Linear(num_ftrs, 128), # First fully connected layer\n",
+ " nn.ReLU(), # Activation function\n",
+ " nn.Dropout(0.4), # Dropout after the first custom layer\n",
+ " nn.Linear(128, 2) # Output layer for binary classification\n",
+ ")\n",
+ "\n",
"model = model.to(device)\n",
+ "\n",
+ "\n",
"# Set the loss function\n",
"criterion = nn.CrossEntropyLoss()\n",
"\n",
@@ -3977,14 +3967,40 @@
" model, criterion, optimizer_conv, exp_lr_scheduler, num_epochs=10\n",
")\n",
"\n",
+ "model.eval()\n",
+ "print_size_of_model(model, \"fp32\")\n",
+ "quantized_model = torch.quantization.convert(model, inplace=False)\n",
+ "\n",
"# Evaluate the model on the test set\n",
- "test_loss, test_acc = eval_model(model, criterion, dataloaders[\"test\"], dataset_sizes[\"test\"])"
+ "test_loss, test_acc = eval_model(model, criterion, dataloaders[\"test\"], dataset_sizes[\"test\"])\n",
+ "\n",
+ "print_size_of_model(quantized_model, \"int8\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "14e5a685",
+ "metadata": {},
+ "source": [
+ "It can be observed that after applying post-training quantization, the model size does not change significantly compared to the original size.\n",
+ "\n",
+ "This may be because ResNet18 is a relatively small model, with a moderate number of parameters compared to larger architectures. As a result, the reduction in model size is limited.\n",
+ "\n",
+ "Additionally, post-training quantization is typically applied to layers that have a large number of parameters, such as Linear and Conv2d layers. However, in models like ResNet18, many of the layers are small or have fewer parameters, which can reduce the impact of quantization on the overall model size."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1891c5df",
+ "metadata": {},
+ "source": [
+ "#### b. Quantization-Aware Training"
]
},
{
"cell_type": "code",
- "execution_count": 156,
- "id": "1ad646f0",
+ "execution_count": 162,
+ "id": "4c282cca",
"metadata": {},
"outputs": [
{
@@ -4009,7 +4025,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/heber/.pyenv/versions/3.11.7/lib/python3.11/site-packages/torch/ao/quantization/quantize.py:392: UserWarning: None of the submodule got qconfig applied. Make sure you passed correct configuration through `qconfig_dict` or by assigning the `.qconfig` attribute directly on submodules\n",
+ "/Users/heber/.pyenv/versions/3.11.7/lib/python3.11/site-packages/torch/ao/quantization/observer.py:229: UserWarning: Please use quant_min and quant_max to specify the range for observers. reduce_range will be deprecated in a future release of PyTorch.\n",
" warnings.warn(\n",
"/Users/heber/.pyenv/versions/3.11.7/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:224: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`. Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
" warnings.warn(\n"
@@ -4019,68 +4035,68 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "train Loss: 0.7334 Acc: 0.5205\n",
- "val Loss: 0.6306 Acc: 0.7255\n",
+ "train Loss: 0.7774 Acc: 0.5123\n",
+ "val Loss: 0.9556 Acc: 0.4575\n",
"\n",
"Epoch 2/10\n",
"----------\n",
- "train Loss: 0.6679 Acc: 0.5697\n",
- "val Loss: 0.5974 Acc: 0.7712\n",
+ "train Loss: 0.7116 Acc: 0.5615\n",
+ "val Loss: 0.6130 Acc: 0.6601\n",
"\n",
"Epoch 3/10\n",
"----------\n",
- "train Loss: 0.6224 Acc: 0.6680\n",
- "val Loss: 0.5652 Acc: 0.7778\n",
+ "train Loss: 0.6686 Acc: 0.5984\n",
+ "val Loss: 0.5965 Acc: 0.6993\n",
"\n",
"Epoch 4/10\n",
"----------\n",
- "train Loss: 0.5957 Acc: 0.7008\n",
- "val Loss: 0.5323 Acc: 0.9346\n",
+ "train Loss: 0.6544 Acc: 0.5902\n",
+ "val Loss: 0.5669 Acc: 0.7582\n",
"\n",
"Epoch 5/10\n",
"----------\n",
- "train Loss: 0.5765 Acc: 0.7377\n",
- "val Loss: 0.4919 Acc: 0.9216\n",
+ "train Loss: 0.6820 Acc: 0.5984\n",
+ "val Loss: 0.5597 Acc: 0.7647\n",
"\n",
"Epoch 6/10\n",
"----------\n",
- "train Loss: 0.5248 Acc: 0.7500\n",
- "val Loss: 0.4573 Acc: 0.9281\n",
+ "train Loss: 0.6664 Acc: 0.6148\n",
+ "val Loss: 0.5594 Acc: 0.7647\n",
"\n",
"Epoch 7/10\n",
"----------\n",
- "train Loss: 0.5351 Acc: 0.7418\n",
- "val Loss: 0.4232 Acc: 0.9346\n",
+ "train Loss: 0.6388 Acc: 0.6311\n",
+ "val Loss: 0.5525 Acc: 0.7908\n",
"\n",
"Epoch 8/10\n",
"----------\n",
- "train Loss: 0.5073 Acc: 0.7828\n",
- "val Loss: 0.3931 Acc: 0.9281\n",
+ "train Loss: 0.6335 Acc: 0.6639\n",
+ "val Loss: 0.5499 Acc: 0.7647\n",
"\n",
"Epoch 9/10\n",
"----------\n",
- "train Loss: 0.4724 Acc: 0.7828\n",
- "val Loss: 0.3696 Acc: 0.9412\n",
+ "train Loss: 0.6635 Acc: 0.6148\n",
+ "val Loss: 0.5490 Acc: 0.7778\n",
"\n",
"Epoch 10/10\n",
"----------\n",
- "train Loss: 0.4548 Acc: 0.8238\n",
- "val Loss: 0.3533 Acc: 0.9346\n",
+ "train Loss: 0.6180 Acc: 0.6598\n",
+ "val Loss: 0.5432 Acc: 0.7843\n",
"\n",
- "Training complete in 7m 44s\n",
- "Best val Acc: 0.941176\n",
- "Test Loss: 0.3277 Acc: 0.9592\n",
- "model: fp32 \t Size (KB): 45040.57\n",
- "model: int8 \t Size (KB): 45040.57\n"
+ "Training complete in 11m 36s\n",
+ "Best val Acc: 0.790850\n",
+ "model: fp32 \t Size (KB): 45213.737\n",
+ "Test Loss: 0.5946 Acc: 0.7755\n",
+ "model: int8 \t Size (KB): 11430.801\n"
]
},
{
"data": {
"text/plain": [
- "45040570"
+ "11430801"
]
},
- "execution_count": 156,
+ "execution_count": 162,
"metadata": {},
"output_type": "execute_result"
}
@@ -4099,6 +4115,9 @@
"from torch.optim import lr_scheduler\n",
"from torchvision import datasets, transforms\n",
"\n",
+ "device = torch.device(\"cpu\")\n",
+ "os.environ[\"PYTORCH_ENABLE_MPS_FALLBACK\"] = \"1\"\n",
+ "\n",
"\n",
"def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n",
" since = time.time()\n",
@@ -4182,11 +4201,18 @@
"model = torchvision.models.resnet18(pretrained=True)\n",
"\n",
"# Freeze the earlier layers for fine-tuning\n",
- "for param in model.parameters():\n",
- " param.requires_grad = False\n",
+ "# for param in model.parameters():\n",
+ "# param.requires_grad = False\n",
"\n",
- "num_ftrs = model.fc.in_features\n",
+ "model.eval()\n",
+ "model = torch.quantization.fuse_modules(\n",
+ " model,\n",
+ " [[\"conv1\", \"bn1\", \"relu\"]] +\n",
+ " [[f\"layer{i}.{j}.conv1\", f\"layer{i}.{j}.bn1\", f\"layer{i}.{j}.relu\"] for i in range(1, 5) for j in range(len(getattr(model, f\"layer{i}\")))],\n",
+ " inplace=True\n",
+ ")\n",
"\n",
+ "num_ftrs = model.fc.in_features\n",
"model.fc = nn.Sequential(\n",
" nn.Dropout(0.4), # Dropout before the first custom layer\n",
" nn.Linear(num_ftrs, 128), # First fully connected layer\n",
@@ -4195,11 +4221,12 @@
" nn.Linear(128, 2) # Output layer for binary classification\n",
")\n",
"\n",
- "model = model.to(device)\n",
- "\n",
+ "model.qconfig = torch.quantization.get_default_qat_qconfig(\"fbgemm\")\n",
"model.train()\n",
"model = torch.quantization.prepare_qat(model, inplace=False)\n",
"\n",
+ "model = model.to(device)\n",
+ "\n",
"# Set the loss function\n",
"criterion = nn.CrossEntropyLoss()\n",
"\n",
@@ -4214,15 +4241,23 @@
")\n",
"\n",
"model.eval()\n",
+ "print_size_of_model(model, \"fp32\")\n",
"quantized_model = torch.quantization.convert(model, inplace=False)\n",
"\n",
"# Evaluate the model on the test set\n",
"test_loss, test_acc = eval_model(model, criterion, dataloaders[\"test\"], dataset_sizes[\"test\"])\n",
"\n",
- "print_size_of_model(model, \"fp32\")\n",
"print_size_of_model(quantized_model, \"int8\")"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "240239ff",
+ "metadata": {},
+ "source": [
+ "By applying Quantization-Aware Training, a significant reduction in the model size is observed, decreasing from 45213.737 KB to 11430.801 KB. However, this decrease in size comes at the cost of model accuracy, as evidenced by a drop from 95.42% to 79.09% in the best validation accuracy, and from 93.88% to 77.55% in the test accuracy."
+ ]
+ },
{
"cell_type": "markdown",
"id": "04a263f0",
--
GitLab