diff --git a/TD2 Deep Learning.ipynb b/TD2 Deep Learning.ipynb index b7cdeb2e957d162b3c2ae5e42c9c6634fbacc330..9210050fdeb3c7044bba7b36f033b607e9da8c42 100644 --- a/TD2 Deep Learning.ipynb +++ b/TD2 Deep Learning.ipynb @@ -1071,14 +1071,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "ef623c26", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model: fp32 \t Size (KB): 758.082\n" + ] + }, + { + "data": { + "text/plain": [ + "758082" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import os\n", "\n", - "\n", + "# I'm using the model that I has created, so I changed the name from model to new_model\n", "def print_size_of_model(model, label=\"\"):\n", " torch.save(model.state_dict(), \"temp.p\")\n", " size = os.path.getsize(\"temp.p\")\n", @@ -1087,7 +1105,7 @@ " return size\n", "\n", "\n", - "print_size_of_model(model, \"fp32\")" + "print_size_of_model(new_model, \"fp32\")" ] }, { @@ -1100,15 +1118,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "c4c65d4b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model: int8 \t Size (KB): 266.59\n" + ] + }, + { + "data": { + "text/plain": [ + "266590" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import torch.quantization\n", "\n", "\n", - "quantized_model = torch.quantization.quantize_dynamic(model, dtype=torch.qint8)\n", + "quantized_model = torch.quantization.quantize_dynamic(new_model, dtype=torch.qint8)\n", "print_size_of_model(quantized_model, \"int8\")" ] }, @@ -1120,6 +1156,101 @@ "For each class, compare the classification test accuracy of the initial model and the quantized model. Also give the overall test accuracy for both models." ] }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test Loss: 17.602363\n", + "\n", + "Test Accuracy of airplane: 75% (756/1000)\n", + "Test Accuracy of automobile: 85% (852/1000)\n", + "Test Accuracy of bird: 59% (590/1000)\n", + "Test Accuracy of cat: 54% (543/1000)\n", + "Test Accuracy of deer: 70% (709/1000)\n", + "Test Accuracy of dog: 54% (544/1000)\n", + "Test Accuracy of frog: 82% (828/1000)\n", + "Test Accuracy of horse: 70% (703/1000)\n", + "Test Accuracy of ship: 78% (783/1000)\n", + "Test Accuracy of truck: 81% (812/1000)\n", + "\n", + "Test Accuracy (Overall): 71% (7120/10000)\n" + ] + } + ], + "source": [ + "# track test loss\n", + "test_loss = 0.0\n", + "class_correct = list(0.0 for i in range(10))\n", + "class_total = list(0.0 for i in range(10))\n", + "\n", + "quantized_model.eval()\n", + "# iterate over test data\n", + "for data, target in test_loader:\n", + " # move tensors to GPU if CUDA is available\n", + " if train_on_gpu:\n", + " data, target = data.cuda(), target.cuda()\n", + " # forward pass: compute predicted outputs by passing inputs to the quantized_model\n", + " output = quantized_model(data)\n", + " # calculate the batch loss\n", + " loss = criterion(output, target)\n", + " # update test loss\n", + " test_loss += loss.item() * data.size(0)\n", + " # convert output probabilities to predicted class\n", + " _, pred = torch.max(output, 1)\n", + " # compare predictions to true label\n", + " correct_tensor = pred.eq(target.data.view_as(pred))\n", + " correct = (\n", + " np.squeeze(correct_tensor.numpy())\n", + " if not train_on_gpu\n", + " else np.squeeze(correct_tensor.cpu().numpy())\n", + " )\n", + " # calculate test accuracy for each object class\n", + " for i in range(batch_size):\n", + " label = target.data[i]\n", + " class_correct[label] += correct[i].item()\n", + " class_total[label] += 1\n", + "\n", + "# average test loss\n", + "test_loss = test_loss / len(test_loader)\n", + "print(\"Test Loss: {:.6f}\\n\".format(test_loss))\n", + "\n", + "for i in range(10):\n", + " if class_total[i] > 0:\n", + " print(\n", + " \"Test Accuracy of %5s: %2d%% (%2d/%2d)\"\n", + " % (\n", + " classes[i],\n", + " 100 * class_correct[i] / class_total[i],\n", + " np.sum(class_correct[i]),\n", + " np.sum(class_total[i]),\n", + " )\n", + " )\n", + " else:\n", + " print(\"Test Accuracy of %5s: N/A (no training examples)\" % (classes[i]))\n", + "\n", + "print(\n", + " \"\\nTest Accuracy (Overall): %2d%% (%2d/%2d)\"\n", + " % (\n", + " 100.0 * np.sum(class_correct) / np.sum(class_total),\n", + " np.sum(class_correct),\n", + " np.sum(class_total),\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "<p><span style=\"color:orange;\">Answer :</span> Overall accuracy still the same : 71%, \n", + "<br>but we notice that their is a slice difference in the accuracy of each class (but not a big deal for a quantized model), for example : Acc_quantized_model(deer) = 70% ~ Acc_original_model(deer) = 71% .</p>" + ] + }, { "cell_type": "markdown", "id": "a0a34b90",