diff --git a/main.ipynb b/main.ipynb index e1dc11eeb0ddbb56e5a82df007b92a1d9cfd2cd6..72d5d962f2be70c0a1acfd68913a545a2b823724 100644 --- a/main.ipynb +++ b/main.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -61,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -100,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -410,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -420,7 +420,11 @@ " learn_once_mse,\n", " one_hot,\n", " learn_once_cross_entropy,\n", - " run_mlp_training\n", + " run_mlp_training,\n", + " learn_once_mse,\n", + " softmax,\n", + " test_mlp,\n", + " train_mlp,\n", ")" ] }, @@ -433,7 +437,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -454,14 +458,14 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Loss: 0.10367831888711801\n" + "Loss: 0.07218991126027921\n" ] } ], @@ -488,7 +492,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 42, "metadata": {}, "outputs": [ { @@ -499,7 +503,7 @@ " [1., 0., 0.]])" ] }, - "execution_count": 52, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -517,9 +521,17 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(30,)\n" + ] + } + ], "source": [ "N = 30 # number of input data\n", "d_in = 3 # input dimension\n", @@ -533,19 +545,21 @@ "b2 = np.zeros((1, d_out)) # second layer biaises\n", "\n", "random_data = np.random.rand(N, d_in) # create a random data\n", - "random_targets = np.random.randint(1, d_out, N) # create a random targets" + "random_targets = np.random.randint(1, d_out, (N)) # create a random targets\n", + "\n", + "print(random_targets.shape)" ] }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 44, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Loss: 0.6940785845571713\n" + "Loss: 0.7075029802848043\n" ] } ], @@ -572,7 +586,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -583,15 +597,127 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/home/tracert6/Documents/ECL-S9-DeepLearning/TP/TP1/mlp.py:210: RuntimeWarning: overflow encountered in exp\n", - " z1 = np.matmul(a0, w1) + b1 # input of the hidden layer\n" + "/home/tracert6/Documents/ECL-S9-DeepLearning/TP/TP1/mlp.py:221: RuntimeWarning: overflow encountered in exp\n", + " z2 = np.matmul(a1, w2) + b2\n", + "/home/tracert6/Documents/ECL-S9-DeepLearning/TP/TP1/mlp.py:118: RuntimeWarning: overflow encountered in exp\n", + " z2 = np.matmul(a1, w2) + b2\n", + "/home/tracert6/Documents/ECL-S9-DeepLearning/TP/TP1/mlp.py:16: RuntimeWarning: invalid value encountered in divide\n", + " \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0: 0.10592592592592592\n", + "Epoch 1: 0.09855555555555555\n", + "Epoch 2: 0.10437037037037038\n", + "Epoch 3: 0.109\n", + "Epoch 4: 0.10901851851851851\n", + "Epoch 5: 0.10907407407407407\n", + "Epoch 6: 0.10044444444444445\n", + "Epoch 7: 0.0995925925925926\n", + "Epoch 8: 0.09985185185185186\n", + "Epoch 9: 0.09985185185185186\n", + "Epoch 10: 0.09985185185185186\n", + "Epoch 11: 0.09985185185185186\n", + "Epoch 12: 0.09985185185185186\n", + "Epoch 13: 0.09985185185185186\n", + "Epoch 14: 0.09985185185185186\n", + "Epoch 15: 0.09985185185185186\n", + "Epoch 16: 0.09985185185185186\n", + "Epoch 17: 0.09985185185185186\n", + "Epoch 18: 0.09985185185185186\n", + "Epoch 19: 0.09985185185185186\n", + "Epoch 20: 0.09985185185185186\n", + "Epoch 21: 0.09985185185185186\n", + "Epoch 22: 0.09985185185185186\n", + "Epoch 23: 0.09985185185185186\n", + "Epoch 24: 0.09985185185185186\n", + "Epoch 25: 0.09985185185185186\n", + "Epoch 26: 0.09985185185185186\n", + "Epoch 27: 0.09985185185185186\n", + "Epoch 28: 0.09985185185185186\n", + "Epoch 29: 0.09985185185185186\n", + "Epoch 30: 0.09985185185185186\n", + "Epoch 31: 0.09985185185185186\n", + "Epoch 32: 0.09985185185185186\n", + "Epoch 33: 0.09985185185185186\n", + "Epoch 34: 0.09985185185185186\n", + "Epoch 35: 0.09985185185185186\n", + "Epoch 36: 0.09985185185185186\n", + "Epoch 37: 0.09985185185185186\n", + "Epoch 38: 0.09985185185185186\n", + "Epoch 39: 0.09985185185185186\n", + "Epoch 40: 0.09985185185185186\n", + "Epoch 41: 0.09985185185185186\n", + "Epoch 42: 0.09985185185185186\n", + "Epoch 43: 0.09985185185185186\n", + "Epoch 44: 0.09985185185185186\n", + "Epoch 45: 0.09985185185185186\n", + "Epoch 46: 0.09985185185185186\n", + "Epoch 47: 0.09985185185185186\n", + "Epoch 48: 0.09985185185185186\n", + "Epoch 49: 0.09985185185185186\n", + "Epoch 50: 0.09985185185185186\n", + "Epoch 51: 0.09985185185185186\n", + "Epoch 52: 0.09985185185185186\n", + "Epoch 53: 0.09985185185185186\n", + "Epoch 54: 0.09985185185185186\n", + "Epoch 55: 0.09985185185185186\n", + "Epoch 56: 0.09985185185185186\n", + "Epoch 57: 0.09985185185185186\n", + "Epoch 58: 0.09985185185185186\n", + "Epoch 59: 0.09985185185185186\n", + "Epoch 60: 0.09985185185185186\n", + "Epoch 61: 0.09985185185185186\n", + "Epoch 62: 0.09985185185185186\n", + "Epoch 63: 0.09985185185185186\n", + "Epoch 64: 0.09985185185185186\n", + "Epoch 65: 0.09985185185185186\n", + "Epoch 66: 0.09985185185185186\n", + "Epoch 67: 0.09985185185185186\n", + "Epoch 68: 0.09985185185185186\n", + "Epoch 69: 0.09985185185185186\n", + "Epoch 70: 0.09985185185185186\n", + "Epoch 71: 0.09985185185185186\n", + "Epoch 72: 0.09985185185185186\n", + "Epoch 73: 0.09985185185185186\n", + "Epoch 74: 0.09985185185185186\n", + "Epoch 75: 0.09985185185185186\n", + "Epoch 76: 0.09985185185185186\n", + "Epoch 77: 0.09985185185185186\n", + "Epoch 78: 0.09985185185185186\n", + "Epoch 79: 0.09985185185185186\n", + "Epoch 80: 0.09985185185185186\n", + "Epoch 81: 0.09985185185185186\n", + "Epoch 82: 0.09985185185185186\n", + "Epoch 83: 0.09985185185185186\n", + "Epoch 84: 0.09985185185185186\n", + "Epoch 85: 0.09985185185185186\n", + "Epoch 86: 0.09985185185185186\n", + "Epoch 87: 0.09985185185185186\n", + "Epoch 88: 0.09985185185185186\n", + "Epoch 89: 0.09985185185185186\n", + "Epoch 90: 0.09985185185185186\n", + "Epoch 91: 0.09985185185185186\n", + "Epoch 92: 0.09985185185185186\n", + "Epoch 93: 0.09985185185185186\n", + "Epoch 94: 0.09985185185185186\n", + "Epoch 95: 0.09985185185185186\n", + "Epoch 96: 0.09985185185185186\n", + "Epoch 97: 0.09985185185185186\n", + "Epoch 98: 0.09985185185185186\n", + "Epoch 99: 0.09985185185185186\n", + "Epoch 100: 0.09985185185185186\n", + "Test accuracy: 0.10133333333333333\n" ] } ], @@ -614,9 +740,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGwCAYAAABSN5pGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/yklEQVR4nO3df3RU1b3//9f8yoQAATSSAAIRQRFUQH4JekuLqaCsWkuq0qJQ7BdKJYCkWqGKqK0GMUJq5UrtEm/vp72C2OpFe6ULg2hBBARR5JdWbaGEBChCgEgSZs73j2R+ZmYyMzmTmSTPx1qzSmYOe3YOq57X2vu997YYhmEIAAAAXtZkdwAAACDVEJAAAACCEJAAAACCEJAAAACCEJAAAACCEJAAAACCEJAAAACC2JPdgZbK7XarrKxMHTt2lMViSXZ3AABAFAzD0OnTp9W9e3dZreHHiQhIcSorK1PPnj2T3Q0AABCHQ4cO6eKLLw77OQEpTh07dpRUd4MzMzOT3BsAABCNyspK9ezZ0/scD4eAFCfPtFpmZiYBCQCAFqax8hiKtAEAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkAAAAIIQkNqQ6vMuVZ93JbsbAACkPHuyO4DEMgxDHx46qVXbDuqNj4+oW6d0rZ83RlZr5FOMAQBoywhIrYxhGDp2ulqHvqrSrkOn9PL2QzpQcdr7+efHzurceZcy0vinBwAgHJ6SrcST6/Zr/d4K/eurKp2rdQd85rRbddOVOXptV5kk6bzbSEYXAQBoMQhIrcCpqlo9t/Fz789Wi9StUzv1vjBD46/M0XcH91BHp90bkFwuAhIAAJEQkFqBr2vrCq/tVoveKhyj7p3bKc3esP7eYpEMQ6p1uxt8BgAAfAhIrYBnZZrTblVuVvuw19mtFtW6DLmYYgMAICKW+bcC1efrRoScDlvE62z1K9fOM8UGAEBEBKRWoLq+KNsZYlrNn8Na9zkjSAAAREZAagX8p9gisdnqR5AISAAARERAagVqPFNs9shTbHbPFBtF2gAARERAagV8NUiNjCBRgwQAQFQISK2AZ4otzRb5n9NODRIAAFEhILUC0Y4g2alBAgAgKgSkVsC3ii26Zf6MIAEAEBkBqRWodkW3zN9bpO2iSBsAgEgISK1AdW2Uy/zra5CYYgMAIDICUivgqUEKdf6aPztTbAAARIWA1ApUR7kPkneZPwEJAICICEitQLQ7aTtsnhEkapAAAIiEgNQK1MS6USQjSAAARERAagWinWLzbBTJTtoAAERGQGoFPPsgNVakzQgSAADRISC1AtHWIPlWsVGDBABAJASkViDqKTaOGgEAICoEpFbAW6Td6AgSh9UCABANAlIr4J1ii3IVWy1F2gAARERAagW8O2nbqEECAMAMBKRWwLOKzelgJ20AAMxAQGoFol7FVj/C5GKKDQCAiAhIrUCNK9oibUaQAACIBgGpFfBOsUV9WC01SAAAREJAagW8RdqMIAEAYIqkB6Tly5crNzdX6enpGjlypLZt2xb22j179ig/P1+5ubmyWCwqKSmJq83y8nLdddddysnJUfv27XXNNdfoT3/6k5m/VrOKtgbJVr9RJDVIAABEltSAtHr1ahUWFmrRokXauXOnBg0apHHjxuno0aMhr6+qqlKfPn20ePFi5eTkxN3mlClTdODAAa1du1a7d+/WxIkTdfvtt+vDDz9MyO+ZSIZh+HbSbmQfJEaQAACITlID0tKlSzV9+nRNmzZNAwYM0IoVK5SRkaGVK1eGvH748OF66qmnNGnSJDmdzrjbfO+99zR79myNGDFCffr00UMPPaTOnTtrx44dYftaXV2tysrKgFcqOO82ZNTnnUaPGmEnbQAAopK0gFRTU6MdO3YoLy/P1xmrVXl5edqyZUtC2xw9erRWr16tEydOyO12a9WqVTp37py++c1vhm27qKhInTp18r569uwZVx/N5hk9kljFBgCAWZIWkI4fPy6Xy6Xs7OyA97Ozs1VeXp7QNl9++WXV1tbqwgsvlNPp1E9+8hO9+uqr6tu3b9i2FyxYoFOnTnlfhw4diquPZquudXn/3NhO2p4apPMuVrEBABCJPdkdSIaFCxfq5MmTeuutt5SVlaXXXntNt99+u/72t7/pqquuCvl3nE5n2Gm9ZPI/ZsRaP0IUju+oEUaQAACIJGkBKSsrSzabTRUVFQHvV1RUhC3ANqPNzz//XM8++6w++eQTDRw4UJI0aNAg/e1vf9Py5cu1YsWKuL47WbwF2o1Mr0mSrb4GiSk2AAAiS9oUW1pamoYOHarS0lLve263W6WlpRo1alTC2qyqqpJUV5vkz2azyd0CN1CsiXIFmyQ5bIwgAQAQjaROsRUWFmrq1KkaNmyYRowYoZKSEp09e1bTpk2TVLccv0ePHioqKpJUV4S9d+9e758PHz6sXbt2qUOHDt76ocba7N+/v/r27auf/OQnKi4u1oUXXqjXXntN69ev1xtvvJGEu9A0vj2QIq9gk9hJGwCAaCU1IN1xxx06duyYHn74YZWXl2vw4MFat26dt8j64MGDASM9ZWVlGjJkiPfn4uJiFRcXa8yYMdq4cWNUbTocDv3f//2f5s+fr+985zs6c+aM+vbtq9///ve6+eabm++XN0m0u2hLfqvY2CgSAICILIZh8LSMQ2VlpTp16qRTp04pMzMzaf3Y9Nlx3fnCVvXP6ah1934j4rWv7PiX7lvzkcZcdpF+f/eIZuohAACpI9rnd9KPGkHTRHvMiMQqNgAAokVAauG8RdpR1CDZbdQgAQAQDQJSCxftOWwSI0gAAESLgNTCeabYGttFW/Ltg1RLkTYAABERkFo4RpAAADAfAamFq66NvgbJxmG1AABEhYDUwtW4oj9qxDeCRJE2AACREJBauOraGJb52ziLDQCAaBCQWrhYdtK2UYMEAEBUCEgtXHUs+yBx1AgAAFEhILVwseykzWG1AABEh4DUwsW0zN/GFBsAANEgILVwsU2xUaQNAEA0CEgtnGcfpGiKtL3L/KlBAgAgIgJSCxdPDVItNUgAAEREQGrhYppiowYJAICoEJBauJrz0e+kzVEjAABEh4DUwsWyis1RX6RtGJKbkAQAQFgEpBbOU4OUZotiBKl+ik1iFAkAgEgISC2cZxWb0xH9TtoSm0UCABAJAamFq46jBkliBAkAgEgISC1cTQzL/D0bRUrshQQAQCQEpBbOV6Td+BSb3wASI0gAAERAQGrBDMPwBqRoirQtFosc7IUEAECjCEgtWI3LV2gdzTJ/yX8vJIq0AQAIh4DUgnlGj6ToapAkvwNrqUECACAsAlILVuMXkKKZYpPYTRsAgGgQkFow/yX+FoulkavrePZCogYJAIDwCEgtWHVt/S7aUU6vSb4Da6lBAgAgPAJSC+YbQWp8ib+HpwaJESQAAMIjILVgseyi7eGpQaqlSBsAgLAISC2YZ4ot2iX+EjVIAABEg4DUgnn2QYplio19kAAAaBwBqQWrrq3fRTumIm1qkAAAaAwBqQWLpwbJzj5IAAA0ioDUglWfr69BiqNIm520AQAIj4DUgsW3zN9TpE0NEgAA4RCQWjDPUSOxrGLjqBEAABpHQGrB4pli8+ykTZE2AADhEZBaMM8qttiKtOuupQYJAIDwCEgtWNNqkAhIAACEQ0BqwZqyiq2WIm0AAMIiILVgNfHsg0QNEgAAjSIgtWDeKTZHLEeNUIMEAEBjCEgtmCcgpdmi/2d0UIMEAECjCEgtmLcGiX2QAAAwFQGpBYtrmb/Nc9QIRdoAAIRDQGrBalyxL/NnBAkAgMYRkFqwpmwUSQ0SAADhEZBaME8NUlpMAYkRJAAAGkNAasHi2Unb5t0HiRokAADCISC1YL59kGIfQaplHyQAAMJKekBavny5cnNzlZ6erpEjR2rbtm1hr92zZ4/y8/OVm5sri8WikpKSuNvcsmWLxo4dq/bt2yszM1Pf+MY39PXXX5v1azWLeHbStlGDBABAo5IakFavXq3CwkItWrRIO3fu1KBBgzRu3DgdPXo05PVVVVXq06ePFi9erJycnLjb3LJli8aPH68bb7xR27Zt0/bt21VQUCCrNel5MSa+s9hiP6yWGiQAAMJLaiJYunSppk+frmnTpmnAgAFasWKFMjIytHLlypDXDx8+XE899ZQmTZokp9MZd5vz5s3TnDlzNH/+fA0cOFCXX365br/99rBtpirPKrZYirRtVmqQAABoTNICUk1NjXbs2KG8vDxfZ6xW5eXlacuWLQlr8+jRo9q6dau6du2q0aNHKzs7W2PGjNGmTZsitl1dXa3KysqAV7JVxzHF5rAxggQAQGOSFpCOHz8ul8ul7OzsgPezs7NVXl6esDa/+OILSdIjjzyi6dOna926dbrmmmt0ww036LPPPgvbdlFRkTp16uR99ezZM64+msXtNvw2iqQGCQAAM7WsohsTuOunln7yk59o2rRpGjJkiJYtW6bLL7887NSeJC1YsECnTp3yvg4dOpSQ/v3l4yN67PW9evfTYxGvq/E7KsTpiKMGiVVsAACEZU/WF2dlZclms6mioiLg/YqKirAF2Ga02a1bN0nSgAEDAq654oordPDgwbBtO53OZqlR2vT343pp20F1znDoG5ddFPY6z/SaFOsIkmeKjRokAADCSdoIUlpamoYOHarS0lLve263W6WlpRo1alTC2szNzVX37t114MCBgL/76aefqnfv3nF9r5kcUR4m61nBZrH4RoWiYfcWaTOCBABAOEkbQZKkwsJCTZ06VcOGDdOIESNUUlKis2fPatq0aZKkKVOmqEePHioqKpJUV4S9d+9e758PHz6sXbt2qUOHDurbt29UbVosFt1///1atGiRBg0apMGDB+v3v/+99u/fr1deeSUJdyGQ56y02kYCjP85bBZLDAHJVtc+RdoAAISX1IB0xx136NixY3r44YdVXl6uwYMHa926dd4i64MHDwbsTVRWVqYhQ4Z4fy4uLlZxcbHGjBmjjRs3RtWmJN177706d+6c5s2bpxMnTmjQoEFav369Lr300ub5xSOIfgQp9mNGJEaQAACIRlIDkiQVFBSooKAg5Gee0OORm5srw2j8wR6pTY/58+dr/vz5Ufezudht0R0FEs8u2pKvBqm2kQAGAEBb1uZWsaU6zxRbY0XU3l20YziHra59RpAAAGgMASnF+KbYGqlBqh9BSrPFN4JEDRIAAOERkFKMZyPHxqbY4q1BctjYKBIAgMbEHJDefvvtRPQD9TwjSI2dlVZdG98Um42NIgEAaFTMAWn8+PG69NJL9atf/Sphu0m3ZZ4aocaW+cdzzIh/+2wUCQBAeDEHpMOHD6ugoECvvPKK+vTpo3Hjxunll19WTU1NIvrX5nj3KWpsmX9tfFNs1CABANC4mANSVlaW5s2bp127dmnr1q267LLLdM8996h79+6aM2eOPvroo0T0s82IuUg71hEkG6vYAABoTJOKtK+55hotWLBABQUFOnPmjFauXKmhQ4fqP/7jP7Rnzx6z+timRL2TtmeZf8z7IHlGqAhIAACEE1dAqq2t1SuvvKKbb75ZvXv31l//+lc9++yzqqio0N///nf17t1bt912m9l9bRPs7KQNAEDSxbyT9uzZs/XSSy/JMAzdddddWrJkia688krv5+3bt1dxcbG6d+9uakfbCoctuhEe707asW4UaaMGCQCAxsQckPbu3avf/OY3mjhxopxOZ8hrsrKy2A4gTr5VbFHupM0qNgAATBdzQCotLW28UbtdY8aMiatDbV20I0ieVWyxFml7apBc1CABABBWzDVIRUVFWrlyZYP3V65cqSeffNKUTrVl0R4m29QaJKbYAAAIL+aA9Nvf/lb9+/dv8P7AgQO1YsUKUzrVlkVbIxT3FBvL/AEAaFTMAam8vFzdunVr8P5FF12kI0eOmNKptizas9K8RdoxT7FRgwQAQGNiDkg9e/bU5s2bG7y/efNmVq6ZwB7rFJsj1im2un9ytyG5GUUCACCkmIu0p0+frnvvvVe1tbUaO3aspLrC7Z///Of62c9+ZnoH25qoi7Q9AckW3wiSVDeNl+b3MwAAqBNzQLr//vv173//W/fcc4/3/LX09HQ98MADWrBggekdbGt8NUhRLvOPdR8kv0BEHRIAAKHFHJAsFouefPJJLVy4UPv27VO7du3Ur1+/sHsiITbeo0aiXOYfb5G25AlhsU3RAQDQFsQckDw6dOig4cOHm9kXyP+w2sgjSDWueJf5+wIVI0gAAIQWV0D64IMP9PLLL+vgwYPeaTaPP//5z6Z0rK2y26I8rDbOEST/kqPGRqkAAGirYl7FtmrVKo0ePVr79u3Tq6++qtraWu3Zs0cbNmxQp06dEtHHNsVhjfaw2roapFh30rZYLBxYCwBAI2IOSE888YSWLVum119/XWlpafr1r3+t/fv36/bbb1evXr0S0cc2xbPKrLFl+PHupO3/HeyFBABAaDEHpM8//1wTJkyQJKWlpens2bOyWCyaN2+enn/+edM72NbY/ZbtRzqw1rcPUsz/hIwgAQDQiJifrl26dNHp06clST169NAnn3wiSTp58qSqqqrM7V0b5PBfZRahRijenbQlXwjjPDYAAEKLuUj7G9/4htavX6+rrrpKt912m+bOnasNGzZo/fr1uuGGGxLRxzbFf5VZpADjO4st9ik2RpAAAIgs5oD07LPP6ty5c5KkBx98UA6HQ++9957y8/P10EMPmd7BtiZwBCn0FJvLbXhXoMVapC35apAaO84EAIC2KqaAdP78eb3xxhsaN26cJMlqtWr+/PkJ6VhbZbFYZLNa5HIbYUeQPNNrUpxTbIwgAQAQUUxPV7vdrpkzZ3pHkJAYjR1Y65lek+ILSDbvcSYEJAAAQon56TpixAjt2rUrAV2BR2MH1npGkGxWS8Cqt6jbr69zYgQJAIDQYq5Buueee1RYWKhDhw5p6NChat++fcDnV199tWmda6saO7C2ugkr2CS/fZDYSRsAgJBiDkiTJk2SJM2ZM8f7nsVikWEYslgscrlc4f4qotTYgbXx7qLtwUaRAABEFnNA+vLLLxPRD/ixNzLCcy7Oc9i87VODBABARDEHpN69eyeiH/DjCTDhdtJuyjEjkmTz1CAxxQYAQEgxB6T//u//jvj5lClT4u4M6jRWpO3bJDK+ESTvgbiMIAEAEFLMAWnu3LkBP9fW1qqqqkppaWnKyMggIJnA3kiNUE0TzmGTfDVIrGIDACC0mJ+wX331VcDrzJkzOnDggK6//nq99NJLiehjm2NvdASpLiClxbHEv659irQBAIgkvidskH79+mnx4sUNRpcQH0fUy/ybVoPEMn8AAEIzJSBJdbtsl5WVmdVcm+bbSTvMCFJtfQ1SnFNsHDUCAEBkMdcgrV27NuBnwzB05MgRPfvss7ruuutM61hbFu0UW5M3iiQgAQAQUswB6dZbbw342WKx6KKLLtLYsWP19NNPm9WvNq2xKbaaJk6xedp3UYMEAEBIMQckNw/VhGt8J+2mjiDVj1AxggQAQEim1SDBPL6dtMMVaTftqJHGduoGAKCti/kJm5+fryeffLLB+0uWLNFtt91mSqfaOt9O2o2NIMW7io0aJAAAIok5IL377ru6+eabG7x/00036d133zWlU22dr0g7zAhSbdM2ivStYmO6FACAUGJ+wp45c0ZpaWkN3nc4HKqsrDSlU22do5EpsBpX3RSbo8kbRTKCBABAKDE/Ya+66iqtXr26wfurVq3SgAEDTOlUW+cdQQoTYGrP170fb5G2pwicfZAAAAgt5lVsCxcu1MSJE/X5559r7NixkqTS0lK99NJLWrNmjekdbIu8y/zDTLHVupp21IitkY0oAQBo62IOSN/5znf02muv6YknntArr7yidu3a6eqrr9Zbb72lMWPGJKKPbY53mX+YEZ6a+oDkCVKxt08NEgAAkcQckCRpwoQJmjBhgtl9QT17IyNIno0iHeykDQBAQsT8hN2+fbu2bt3a4P2tW7fqgw8+MKVTbZ2jsRok7whSvEXa1CABABBJzE/YWbNm6dChQw3eP3z4sGbNmmVKp9o632G14WqQmlqkzQgSAACRxPyE3bt3r6655poG7w8ZMkR79+41pVNtXWM7Xdc0cQTJ1shO3QAAtHUxP2GdTqcqKioavH/kyBHZ7XGVNGn58uXKzc1Venq6Ro4cqW3btoW9ds+ePcrPz1dubq4sFotKSkqa1KZhGLrppptksVj02muvxdV/s/mW+UdexRb3FBsjSAAARBTzE/bGG2/UggULdOrUKe97J0+e1C9+8Qt9+9vfjrkDq1evVmFhoRYtWqSdO3dq0KBBGjdunI4ePRry+qqqKvXp00eLFy9WTk5Ok9ssKSmRxRLfarBE8R41Em4E6XzTVrHZvKvYCEgAAIQSc0AqLi7WoUOH1Lt3b33rW9/St771LV1yySUqLy/X008/HXMHli5dqunTp2vatGkaMGCAVqxYoYyMDK1cuTLk9cOHD9dTTz2lSZMmyel0NqnNXbt26emnnw77XcnisEY+aqSp+yAxggQAQGQxP2F79Oihjz/+WEuWLNGAAQM0dOhQ/frXv9bu3bvVs2fPmNqqqanRjh07lJeX5+uQ1aq8vDxt2bIl1q7F1GZVVZV++MMfavny5WFHovxVV1ersrIy4JUojR1W6xlZSou3SNuzio2NIgEACCmuoqH27dtrxowZAe/t27dPL7zwgoqLi6Nu5/jx43K5XMrOzg54Pzs7W/v374+na1G3OW/ePI0ePVrf/e53o2q3qKhIjz76aFx9ilVjAcY3xcYIEgAAiRDfE7be2bNn9cILL2j06NEaOHCg1q1bZ1a/Emrt2rXasGFD2ALvUDx1V55XqK0OzOI9rDZMkbZpq9jYSRsAgJDiesJu3rxZd999t7KzszVjxgyNHj1ae/fu1SeffBJTO1lZWbLZbA1WxVVUVEQ17RVvmxs2bNDnn3+uzp07y263e1ff5efn65vf/GbIdp1OpzIzMwNeieIZQQpXpO2tQbLHedSIjSJtAAAiiTogHT16VEuWLFH//v31/e9/X507d9bGjRtltVp19913q3///jF/eVpamoYOHarS0lLve263W6WlpRo1alTM7UXb5vz58/Xxxx9r165d3pckLVu2TC+++GJc32sm72G14Zb5N3GKzeYtAicgAQAQStQ1SL1799b3v/99/frXv9a3v/1tWa1Nmp3zKiws1NSpUzVs2DCNGDFCJSUlOnv2rKZNmyZJmjJlinr06KGioiJJdUXYng0pa2pqdPjwYe3atUsdOnRQ3759o2ozJycn5AhVr169dMkll5jyezWF97DasCNITSvSdrDMHwCAiGIKSJs2bVKvXr3Uu3fvuEaMQrnjjjt07NgxPfzwwyovL9fgwYO1bt06b5H1wYMHA8JYWVmZhgwZ4v25uLhYxcXFGjNmjDZu3BhVm6ku0k7XhmFQgwQAQIJFHZD279+vzZs364UXXtDw4cN12WWX6c4775SkJm+0WFBQoIKCgpCfeUKPR25urgyj8ZGPSG2GEk2bzcU3xdawT/6jSvEfVssqNgAAIonpCXvddddp5cqVOnLkiGbOnKk1a9bI5XLpnnvu0e9+9zsdO3YsUf1sUyIVafsfYBvvRpHUIAEAEFlcT9gOHTpo+vTpeu+997Rnzx4NHTpUDz30kLp37252/9okR4QpNv+AFO9RI3ZqkAAAiKjJldZXXHGFiouLdfjwYa1evdqMPrV5vsNqGwYYT/2R1eK7LlbUIAEAEJk5S9Ek2e12TZw40azm2jTfYbWhRpDqQlO89Ud1f5cRJAAAIjEtIME8nsNqQwUYzzEj8dYfSb4apHDbCAAA0NYRkFKQbwQpfJG2I849kCRqkAAAaAwBKQVF2knbnBEklvkDABAJASkF2SMsw/eNIMW/95RvBIkibQAAQol6o0iP733veyE3hrRYLEpPT1ffvn31wx/+UJdffrkpHWyLIhVp1zTxHLa69sOvkgMAAHGMIHXq1EkbNmzQzp07ZbFYZLFY9OGHH2rDhg06f/68Vq9erUGDBmnz5s2J6G+b4B1BirCTdlOm2KhBAgAgsphHkHJycvTDH/5Qzz77rPeMNLfbrblz56pjx45atWqVZs6cqQceeECbNm0yvcNtgd1vGb5hGAEjdrVNPIdN8j/rjYAEAEAoMT9lX3jhBd17770BB8harVbNnj1bzz//vCwWiwoKCvTJJ5+Y2tG2xOF3b4NXsnk2ikwzYRUbG0UCABBazE/Z8+fPa//+/Q3e379/v1wulyQpPT29yQfYtmV2vyNEgkOMbwQp/vvrGUFyG5KbaTYAABqIeYrtrrvu0o9//GP94he/0PDhwyVJ27dv1xNPPKEpU6ZIkt555x0NHDjQ3J62If4BqcEIkolF2pLkMgxZRZgFAMBfzAFp2bJlys7O1pIlS1RRUSFJys7O1rx58/TAAw9Ikm688UaNHz/e3J62If5TbMGF1J4RJDOKtD3tO2xxNwUAQKsUc0Cy2Wx68MEH9eCDD6qyslKSlJmZGXBNr169zOldG2W1WmS11E2BnQ9a6l9jwllsNmtg0Xc6CQkAgAAxByR/wcEI5rHbrKo571Zt8AjSefOKtCWW+gMAEErMT9mKigrddddd6t69u+x2u2w2W8AL5nB4l+KHK9I2ZwSJzSIBAGgo5hGkH/3oRzp48KAWLlyobt26sVotQeoKqV1hi7TTmnDUiMVikc1qkcttMIIEAEAIMQekTZs26W9/+5sGDx6cgO7AI9xeRWaMIHnad7kNRpAAAAgh5qdsz549ZRg8VBPNs9Q/eLfrGhOOGpH8AliI894AAGjrYn7KlpSUaP78+frHP/6RgO7Aw3MeW/CBtd4RpCYUaUt+x40wggQAQAMxT7Hdcccdqqqq0qWXXqqMjAw5HI6Az0+cOGFa59oyz07ZwQHGtCm2+r9PDRIAAA3FHJBKSkoS0A0E8wSY4BEkb5F2E44akTiwFgCASGIOSFOnTk1EPxDEHibA1Jg0guTZRoARJAAAGooqIFVWVno3hfTsnh0Om0eawxFmCsyz7L8pG0VKks0WepUcAACIMiB16dJFR44cUdeuXdW5c+eQex8ZhiGLxSKXy2V6J9sizyq2BkXaJhxWK/mKwCnSBgCgoagC0oYNG3TBBRdIkt5+++2Edgh1HGECTI0Jh9VK1CABABBJVAFpzJgxIf+MxAk7guRd5t+0Im07NUgAAIQV12G1J0+e1LZt23T06FG5g2pYpkyZYkrH2rpwIzw1Zk2xUYMEAEBYMQek119/XZMnT9aZM2eUmZkZUI9ksVgISCbxBKBwR400fYqNfZAAAAgn5qfsz372M9199906c+aMTp48qa+++sr7YpNI83imwIIPq/X83NSdtMO1DwAA4ghIhw8f1pw5c5SRkZGI/qCedwQp7EaR5hRpM4IEAEBDMT9lx40bpw8++CARfYEfe6KPGrFSgwQAQDgx1yBNmDBB999/v/bu3aurrrqqwVlst9xyi2mda8t8h9WGWeZv0mG1jCABANBQzAFp+vTpkqTHHnuswWdsFGke72G14Zb5N/EsNl8ROAEJAIBgMQek4GX9SIzwU2z1R42wUSQAAAnTtKcsEsZ3FEjoIm2zapBcBF4AABqIagTpmWee0YwZM5Senq5nnnkm4rVz5swxpWNtnW+KLXQNUlOX+XtHkJhiAwCggagC0rJlyzR58mSlp6dr2bJlYa+zWCwEJJPYQhRpG4Zh2kaRHDUCAEB4UQWkL7/8MuSfkTiOEEeBuNyGjPo80+SARJE2AABhUYOUokIt86/xW9HGYbUAACROXIfV/utf/9LatWt18OBB1dTUBHy2dOlSUzrW1tlDLPOvPe8LM00t0rZ5jxqhSBsAgGAxB6TS0lLdcsst6tOnj/bv368rr7xS//jHP2QYhq655ppE9LFNcoRY5u8/guQZAYoXI0gAAIQX8zDEggULdN9992n37t1KT0/Xn/70Jx06dEhjxozRbbfdlog+tkm+KTa/ESS/XbQtlqYFJJuVGiQAAMKJOSDt27dPU6ZMkSTZ7XZ9/fXX6tChgx577DE9+eSTpnewrQq1zN+sFWz+7TOCBABAQzE/adu3b++tO+rWrZs+//xz72fHjx83r2dtnG+VmW8EybdJZNNGjyR20gYAIJKYa5CuvfZabdq0SVdccYVuvvlm/exnP9Pu3bv15z//Wddee20i+tgm2UNs5OjdJNKEESRf+xRpAwAQLOaAtHTpUp05c0aS9Oijj+rMmTNavXq1+vXrxwo2E3kPkw2YYjMCPmsKapAAAAgvpoDkcrn0r3/9S1dffbWkuum2FStWJKRjbV2oZfiePzubeMyI5NtGwMUUGwAADcT0pLXZbLrxxhv11VdfJao/qBdqmX+tSQfVSpzFBgBAJDE/aa+88kp98cUXiegL/HiW+ftvFFntPai26UXavn2QqEECACBYzAHpV7/6le677z698cYbOnLkiCorKwNeMIdnCsz/qBEzR5A8AamWESQAABqI+kn72GOP6ezZs7r55pv10Ucf6ZZbbtHFF1+sLl26qEuXLurcubO6dOkSVyeWL1+u3Nxcpaena+TIkdq2bVvYa/fs2aP8/Hzl5ubKYrGopKQk5jZPnDih2bNn6/LLL1e7du3Uq1cvzZkzR6dOnYqr/4ngCLHM3xOWzNgHyVbfBjVIAAA0FHWR9qOPPqqZM2fq7bffNrUDq1evVmFhoVasWKGRI0eqpKRE48aN04EDB9S1a9cG11dVValPnz667bbbNG/evLjaLCsrU1lZmYqLizVgwAD985//1MyZM1VWVqZXXnnF1N8vXvYQ+xT576RtWvuMIAEA0EDUAckw6h6kY8aMMbUDS5cu1fTp0zVt2jRJ0ooVK/SXv/xFK1eu1Pz58xtcP3z4cA0fPlySQn4eTZtXXnml/vSnP3mvv/TSS/X444/rzjvv1Pnz52W3x3WGr6k8G0XWhtwo0rwibWqQAABoKKYnbVPP/wpWU1OjHTt2KC8vz9chq1V5eXnasmVLs7Z56tQpZWZmhg1H1dXVzVpvFeqoEd9GkU3/dwi1Sg4AANSJaajksssuazQknThxIur2jh8/LpfLpezs7ID3s7OztX///li61qQ2jx8/rl/+8peaMWNG2HaLior06KOPxtWneNhDbORYa+JO2p6NIjmLDQCAhmIKSI8++qg6deqUqL4kRWVlpSZMmKABAwbokUceCXvdggULVFhYGPD3evbsmbB++UaQGm4UaWoNEkXaAAA0EFNAmjRpUsjC6XhlZWXJZrOpoqIi4P2Kigrl5OQkvM3Tp09r/Pjx6tixo1599VU5HI6w7TqdTjmdzrj6FI9Qh8mauoqNs9gAAAgr6iet2fVHkpSWlqahQ4eqtLTU+57b7VZpaalGjRqV0DYrKyt14403Ki0tTWvXrlV6enr8v0gCOEIUaVcnYB8kptgAAGgo5lVsZissLNTUqVM1bNgwjRgxQiUlJTp79qx3BdqUKVPUo0cPFRUVSaorwt67d6/3z4cPH9auXbvUoUMH9e3bN6o2PeGoqqpKf/jDHwKKri+66CLZbLaE/K6xsIco0jazBslu47BaAADCiToguRM0FXPHHXfo2LFjevjhh1VeXq7Bgwdr3bp13iLrgwcPymr1BYKysjINGTLE+3NxcbGKi4s1ZswYbdy4Mao2d+7cqa1bt0qSN1R5fPnll8rNzU3I7xoL/yJtwzBksVh8O2mbetQIAQkAgGDJ3/BHUkFBgQoKCkJ+5gk9Hrm5uVGNZkVq85vf/GbCRsTM4r+U/7zbkMNm8Y4gOU2sQap1UYMEAECwpj9pkRB2vxDkmWarqf9fapAAAEgsAlKK8gQYyVeo7d1J24Rl/jaOGgEAICwCUopyhBhBMrVIm40iAQAIi4CUomxWizw7K3g2izR1o0iOGgEAICwCUgpzBB034g1IJpzF5ttJmyJtAACCEZBSWPBu2mYWaVODBABAeASkFOaZBvMVabskUYMEAECiEZBSmCcI+Yq0TRxBogYJAICwCEgpzB60maN3o0gTirQd7IMEAEBYBKQU5gg6L63GxMNqbX4BKdV3FQcAoLkRkFKY78Da+hok7z5IZqxi89tniVEkAAACEJBSmG+KLWijSDN20vYLWUyzAQAQiICUwnxTbPU1SOfrgkyaiWex1bVPQAIAwB8BKYX5ptiCNoo0Yydtv4DkchGQAADwR0BKYZ46odoGNUjmFWlLvhEqAABQh4CUwuxBS/F9q9iaXqRtsVjYTRsAgDAISCnMt5N28Fls5vyzEZAAAAiNgJTCfDtpu+VyG/LkGDNqkCS/ESpqkAAACEBASmF2v8NqPaNHkjk1SJL/CBI1SAAA+CMgpTB7fRCqdbu9BdqSeQHJ0w77IAEAEIiAlMIcfsv8PQXa/u83lS1oI0oAAFCHgJTC/Jf51/odM2KxmBOQglfJAQCAOgSkFObdKNJtmLqLtgc1SAAAhEZASmEOq28VW42J57B5MIIEAEBoBKQU5t0Hya8GyawC7br2PWe9EZAAAPBHQEphdr8pMLM3ifRvnxEkAAACEZBSmP8Ij3+Rtll8q9ioQQIAwB8BKYXZ/Zf5e0aQqEECACDhCEgpzL9I27NXkZk1SJzFBgBAaASkFOZ/WG0ii7QZQQIAIBABKYX5H1abyCJtRpAAAAhEQEphoQ6rddjNL9I+T5E2AAABCEgpzHdYrW+KjREkAAASj4CUwnyH1SaqSJsaJAAAQiEgpTDfYbWGas67JCXmqBFGkAAACERASmG+w2p9I0imTrHVt++iBgkAgAAEpBTmv5FjDavYAABoNgSkFOYt0vZb5m/uKjYOqwUAIBQCUgpzhFrmz2G1AAAkHAEphSV6mb/N76w3AADgQ0BKYfYEL/N3eEeQKNIGAMAfASmF+Q6r9SvSNnGZPzVIAACERkBKYb7Dat2qTcBhtZ6C769rXaa1CQBAa0BASmEOvxqhGm+Rtnmr2HpdkCFJ+vL4WdPaBACgNSAgpTC7d4rNt8zfzCm2fl07SpI+qzhjWpsAALQGBKQU5ptiM1Rz3vwi7X5dO0iSDp/8Wmerz5vWLgAALR0BKYWFHEEyMSB1aZ+mrA5OSdLfjzKKBACABwEphfnOYvPbKNLEKTZJuiy7bhTpMwISAABeBKQUFrDM37tRpHlF2pJvmu2zitOmtgsAQEtGQEphvhEkd0KOGpGkvtn1hdqMIAEA4EVASmHeIm2Xoerz5q9ik6TLPCNIRxlBAgDAg4CUwjxTbJK8AcnsEaR+9SNIh058raoaVrIBACARkFKa3a/eyBNezA5IF7RPU1aHNEnS50fZMBIAAClFAtLy5cuVm5ur9PR0jRw5Utu2bQt77Z49e5Sfn6/c3FxZLBaVlJTE1ea5c+c0a9YsXXjhherQoYPy8/NVUVFh5q/VZP5hqKqm7jgQM5f5e/Stn2b7lEJtAAAkpUBAWr16tQoLC7Vo0SLt3LlTgwYN0rhx43T06NGQ11dVValPnz5avHixcnJy4m5z3rx5ev3117VmzRq98847Kisr08SJExPyO8bLbvUfQaoLSJ7z08zk3VGbQm0AACSlQEBaunSppk+frmnTpmnAgAFasWKFMjIytHLlypDXDx8+XE899ZQmTZokp9MZV5unTp3SCy+8oKVLl2rs2LEaOnSoXnzxRb333nt6//33E/a7xsrmF5Bc7rqdtBMxguTZC+nvFGoDACApyQGppqZGO3bsUF5envc9q9WqvLw8bdmyJWFt7tixQ7W1tQHX9O/fX7169Qr7vdXV1aqsrAx4JZrFYgkYRZLMr0GSpL71I0ifciYbAACSkhyQjh8/LpfLpezs7ID3s7OzVV5enrA2y8vLlZaWps6dO0f9vUVFRerUqZP31bNnz7j6Fyt70MaQZi/zl3wjSIe+qtLX9VN5AAC0ZUmfYmspFixYoFOnTnlfhw4dapbv9V/qLyVmBOnCDk5d0D5NhiF9foxRJAAAkhqQsrKyZLPZGqweq6ioCFuAbUabOTk5qqmp0cmTJ6P+XqfTqczMzIBXcwgeQXKYfNSIR182jAQAwMuezC9PS0vT0KFDVVpaqltvvVWS5Ha7VVpaqoKCgoS1OXToUDkcDpWWlio/P1+SdODAAR08eFCjRo1q8u9lJnvQiFEiptikumm2bV+e0GdBdUgnztbo/235p86dZ+oNANC8xg3M0eCenZPy3UkNSJJUWFioqVOnatiwYRoxYoRKSkp09uxZTZs2TZI0ZcoU9ejRQ0VFRZLqirD37t3r/fPhw4e1a9cudejQQX379o2qzU6dOunHP/6xCgsLdcEFFygzM1OzZ8/WqFGjdO211ybhLoTnCC7StiYmIPULU6i98H8/0V8+PpKQ7wQAIJKeXTLabkC64447dOzYMT388MMqLy/X4MGDtW7dOm+R9cGDB2X1CwVlZWUaMmSI9+fi4mIVFxdrzJgx2rhxY1RtStKyZctktVqVn5+v6upqjRs3Tv/5n//ZPL90DPxHkOxWi6zWxEyx9Qux1H/XoZP6y8dHZLFIU67t3WA0CwCARLo8p2PSvttiGIaRtG9vwSorK9WpUyedOnUqofVIY5/eqC+O1R0B0s5h075fjk/I9xw7Xa3hj78li0Xa99h4Oe1WTXr+fW398oTyr7lYT98+KCHfCwBAc4r2+Z30ESRE5j+llqj6I0nK6pCmLhkOfVVVq8+PnVH5qXPa+uUJpdmtKrzxsoR9LwAAqYg5kxTnv4otEUv8PSwWi7cOaf+R01r85n5J0rTrctWjc7uEfS8AAKmIgJTi/HfSTkvQEn+PvvV1SM9s+EyfHT2jTu0cumdM34R+JwAAqYiAlOL8C6MdCZxik6TL6vdC+ue/qyRJs8f2VacMR0K/EwCAVERASnH+I0iJnGKTpH7ZvtUCPTq3012jeif0+wAASFUEpBTnH4rSEh6QOnj/fP+4y+W02xL6fQAApCpWsaW4gCLtBE+xde2Yrun/cYmqaly6ZVD3hH4XAACpjICU4uz+y/wTXKQtSQ9OGJDw7wAAINUxxZbiHM20zB8AAPjwxE1xAavYCEgAADQLnrgpzv+w2kTupA0AAHx44qY4/yLtRK9iAwAAdXjipjib1X+KLfFF2gAAgICU8ijSBgCg+fHETXH+y/wTvQ8SAACowxM3xTmoQQIAoNnxxE1xAUXajCABANAseOKmODtF2gAANDsCUoqjSBsAgObHEzfF+e+kzRQbAADNgyduirNbKdIGAKC58cRNcQ7OYgMAoNnxxE1xNis1SAAANDeeuCkusEibVWwAADQHAlKK81/mT5E2AADNgyduirOzkzYAAM2OJ26Ko0gbAIDmxxM3xfkv8+ewWgAAmgdP3BQXOIJEkTYAAM2BgJTi/GuQnIwgAQDQLHjiprjAw2r55wIAoDnwxE1xHFYLAEDz44mb4thJGwCA5scTN8X5hyJqkAAAaB48cVOcnSk2AACaHU/cFBdYpM0yfwAAmgMBKcUFFGkzxQYAQLPgiZvi7H7TapzFBgBA8+CJm+IcrGIDAKDZ2ZPdAUSW1cGp6/tmqUv7tIAl/wAAIHEISCnOarXoD//fyGR3AwCANoU5GwAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCAEJAAAgCD2ZHegpTIMQ5JUWVmZ5J4AAIBoeZ7bnud4OASkOJ0+fVqS1LNnzyT3BAAAxOr06dPq1KlT2M8tRmMRCiG53W6VlZWpY8eOslgsprVbWVmpnj176tChQ8rMzDStXTTEvW4e3OfmwX1uHtzn5pHI+2wYhk6fPq3u3bvLag1facQIUpysVqsuvvjihLWfmZnJ//maCfe6eXCfmwf3uXlwn5tHou5zpJEjD4q0AQAAghCQAAAAghCQUozT6dSiRYvkdDqT3ZVWj3vdPLjPzYP73Dy4z80jFe4zRdoAAABBGEECAAAIQkACAAAIQkACAAAIQkACAAAIQkBKMcuXL1dubq7S09M1cuRIbdu2LdldatGKioo0fPhwdezYUV27dtWtt96qAwcOBFxz7tw5zZo1SxdeeKE6dOig/Px8VVRUJKnHrcPixYtlsVh07733et/jPpvj8OHDuvPOO3XhhReqXbt2uuqqq/TBBx94PzcMQw8//LC6deumdu3aKS8vT5999lkSe9zyuFwuLVy4UJdcconatWunSy+9VL/85S8Dzu7iPsfn3Xff1Xe+8x11795dFotFr732WsDn0dzXEydOaPLkycrMzFTnzp314x//WGfOnDG9rwSkFLJ69WoVFhZq0aJF2rlzpwYNGqRx48bp6NGjye5ai/XOO+9o1qxZev/997V+/XrV1tbqxhtv1NmzZ73XzJs3T6+//rrWrFmjd955R2VlZZo4cWISe92ybd++Xb/97W919dVXB7zPfW66r776Stddd50cDofefPNN7d27V08//bS6dOnivWbJkiV65plntGLFCm3dulXt27fXuHHjdO7cuST2vGV58skn9dxzz+nZZ5/Vvn379OSTT2rJkiX6zW9+472G+xyfs2fPatCgQVq+fHnIz6O5r5MnT9aePXu0fv16vfHGG3r33Xc1Y8YM8ztrIGWMGDHCmDVrlvdnl8tldO/e3SgqKkpir1qXo0ePGpKMd955xzAMwzh58qThcDiMNWvWeK/Zt2+fIcnYsmVLsrrZYp0+fdro16+fsX79emPMmDHG3LlzDcPgPpvlgQceMK6//vqwn7vdbiMnJ8d46qmnvO+dPHnScDqdxksvvdQcXWwVJkyYYNx9990B702cONGYPHmyYRjcZ7NIMl599VXvz9Hc17179xqSjO3bt3uvefPNNw2LxWIcPnzY1P4xgpQiampqtGPHDuXl5Xnfs1qtysvL05YtW5LYs9bl1KlTkqQLLrhAkrRjxw7V1tYG3Pf+/furV69e3Pc4zJo1SxMmTAi4nxL32Sxr167VsGHDdNttt6lr164aMmSIfve733k///LLL1VeXh5wnzt16qSRI0dyn2MwevRolZaW6tNPP5UkffTRR9q0aZNuuukmSdznRInmvm7ZskWdO3fWsGHDvNfk5eXJarVq69atpvaHw2pTxPHjx+VyuZSdnR3wfnZ2tvbv35+kXrUubrdb9957r6677jpdeeWVkqTy8nKlpaWpc+fOAddmZ2ervLw8Cb1suVatWqWdO3dq+/btDT7jPpvjiy++0HPPPafCwkL94he/0Pbt2zVnzhylpaVp6tSp3nsZ6r8j3OfozZ8/X5WVlerfv79sNptcLpcef/xxTZ48WZK4zwkSzX0tLy9X165dAz632+264IILTL/3BCS0GbNmzdInn3yiTZs2Jbsrrc6hQ4c0d+5crV+/Xunp6cnuTqvldrs1bNgwPfHEE5KkIUOG6JNPPtGKFSs0derUJPeu9Xj55Zf1xz/+Uf/zP/+jgQMHateuXbr33nvVvXt37nMbwhRbisjKypLNZmuwqqeiokI5OTlJ6lXrUVBQoDfeeENvv/22Lr74Yu/7OTk5qqmp0cmTJwOu577HZseOHTp69KiuueYa2e122e12vfPOO3rmmWdkt9uVnZ3NfTZBt27dNGDAgID3rrjiCh08eFCSvPeS/440zf3336/58+dr0qRJuuqqq3TXXXdp3rx5KioqksR9TpRo7mtOTk6DhUvnz5/XiRMnTL/3BKQUkZaWpqFDh6q0tNT7ntvtVmlpqUaNGpXEnrVshmGooKBAr776qjZs2KBLLrkk4POhQ4fK4XAE3PcDBw7o4MGD3PcY3HDDDdq9e7d27drlfQ0bNkyTJ0/2/pn73HTXXXddg20qPv30U/Xu3VuSdMkllygnJyfgPldWVmrr1q3c5xhUVVXJag18PNpsNrndbknc50SJ5r6OGjVKJ0+e1I4dO7zXbNiwQW63WyNHjjS3Q6aWfKNJVq1aZTidTuO//uu/jL179xozZswwOnfubJSXlye7ay3WT3/6U6NTp07Gxo0bjSNHjnhfVVVV3mtmzpxp9OrVy9iwYYPxwQcfGKNGjTJGjRqVxF63Dv6r2AyD+2yGbdu2GXa73Xj88ceNzz77zPjjH/9oZGRkGH/4wx+81yxevNjo3Lmz8b//+7/Gxx9/bHz3u981LrnkEuPrr79OYs9blqlTpxo9evQw3njjDePLL780/vznPxtZWVnGz3/+c+813Of4nD592vjwww+NDz/80JBkLF261Pjwww+Nf/7zn4ZhRHdfx48fbwwZMsTYunWrsWnTJqNfv37GD37wA9P7SkBKMb/5zW+MXr16GWlpacaIESOM999/P9ldatEkhXy9+OKL3mu+/vpr45577jG6dOliZGRkGN/73veMI0eOJK/TrURwQOI+m+P11183rrzySsPpdBr9+/c3nn/++YDP3W63sXDhQiM7O9twOp3GDTfcYBw4cCBJvW2ZKisrjblz5xq9evUy0tPTjT59+hgPPvigUV1d7b2G+xyft99+O+R/k6dOnWoYRnT39d///rfxgx/8wOjQoYORmZlpTJs2zTh9+rTpfbUYht/WoAAAAKAGCQAAIBgBCQAAIAgBCQAAIAgBCQAAIAgBCQAAIAgBCQAAIAgBCQAAIAgBCQAAIAgBCQBMYrFY9NprryW7GwBMQEAC0Cr86Ec/ksViafAaP358srsGoAWyJ7sDAGCW8ePH68UXXwx4z+l0Jqk3AFoyRpAAtBpOp1M5OTkBry5dukiqm/567rnndNNNN6ldu3bq06ePXnnllYC/v3v3bo0dO1bt2rXThRdeqBkzZujMmTMB16xcuVIDBw6U0+lUt27dVFBQEPD58ePH9b3vfU8ZGRnq16+f1q5dm9hfGkBCEJAAtBkLFy5Ufn6+PvroI02ePFmTJk3Svn37JElnz57VuHHj1KVLF23fvl1r1qzRW2+9FRCAnnvuOc2aNUszZszQ7t27tXbtWvXt2zfgOx599FHdfvvt+vjjj3XzzTdr8uTJOnHiRLP+ngBMYABAKzB16lTDZrMZ7du3D3g9/vjjhmEYhiRj5syZAX9n5MiRxk9/+lPDMAzj+eefN7p06WKcOXPG+/lf/vIXw2q1GuXl5YZhGEb37t2NBx98MGwfJBkPPfSQ9+czZ84Ykow333zTtN8TQPOgBglAq/Gtb31Lzz33XMB7F1xwgffPo0aNCvhs1KhR2rVrlyRp3759GjRokNq3b+/9/LrrrpPb7daBAwdksVhUVlamG264IWIfrr76au+f27dvr8zMTB09ejTeXwlAkhCQALQa7du3bzDlZZZ27dpFdZ3D4Qj42WKxyO12J6JLABKIGiQAbcb777/f4OcrrrhCknTFFVfoo48+0tmzZ72fb968WVarVZdffrk6duyo3NxclZaWNmufASQHI0gAWo3q6mqVl5cHvGe325WVlSVJWrNmjYYNG6brr79ef/zjH7Vt2za98MILkqTJkydr0aJFmjp1qh555BEdO3ZMs2fP1l133aXs7GxJ0iOPPKKZM2eqa9euuummm3T69Glt3rxZs2fPbt5fFEDCEZAAtBrr1q1Tt27dAt67/PLLtX//fkl1K8xWrVqle+65R926ddNLL72kAQMGSJIyMjL017/+VXPnztXw4cOVkZGh/Px8LV261NvW1KlTde7cOS1btkz33XefsrKy9P3vf7/5fkEAzcZiGIaR7E4AQKJZLBa9+uqruvXWW5PdFQAtADVIAAAAQQhIAAAAQahBAtAmUE0AIBaMIAEAAAQhIAEAAAQhIAEAAAQhIAEAAAQhIAEAAAQhIAEAAAQhIAEAAAQhIAEAAAT5/wG2mwSx1slWXQAAAABJRU5ErkJggg==", + "text/plain": [ + "<Figure size 640x480 with 1 Axes>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Plot the training accuracy values\n", "plt.plot(range(num_epoch + 1), training_accuracy_values)\n", @@ -625,6 +762,13 @@ "plt.savefig('./results/mlp.png')\n", "plt.show()\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/mlp.py b/mlp.py index 38188f18290a7aa432e57167f5a65d09ba41f970..944df7fc4918ce010493b179bb1911cea67dd881 100644 --- a/mlp.py +++ b/mlp.py @@ -1,5 +1,20 @@ import numpy as np +EPSILON = 1e-8 + + +def softmax(x: np.ndarray) -> np.ndarray: + """Compute the softmax of the given vector. + + Args: + z (np.ndarray): The vector to compute the softmax of. + + Returns: + np.ndarray: The softmax of the given vector. + """ + e_x = np.exp(x - np.max(x)) + return e_x / e_x.sum(axis=0) + def learn_once_mse( w1: np.ndarray, @@ -46,11 +61,11 @@ def learn_once_mse( dC_da2 = 2 * (predictions - targets) / predictions.shape[0] dC_dz2 = dC_da2 * a2 * (1 - a2) dC_dw2 = np.matmul(a1.T, dC_dz2) - dC_db2 = np.sum(dC_dz2, axis=0, keepdims=True) + dC_db2 = np.sum(dC_dz2, axis=0) dC_da1 = np.matmul(dC_dz2, w2.T) dC_dz1 = dC_da1 * a1 * (1 - a1) dC_dw1 = np.matmul(a0.T, dC_dz1) - dC_db1 = np.sum(dC_dz1, axis=0, keepdims=True) + dC_db1 = np.sum(dC_dz1, axis=0) # Update weights and biases w1 -= learning_rate * dC_dw1 @@ -102,7 +117,8 @@ def learn_once_cross_entropy( z1 = np.matmul(a0, w1) + b1 a1 = 1 / (1 + np.exp(-z1)) z2 = np.matmul(a1, w2) + b2 - a2 = np.exp(z2) / np.sum(np.exp(z2), axis=1, keepdims=True) + a2 = softmax(z2) + predictions = a2 one_hot_targets = one_hot(labels_train) @@ -110,8 +126,8 @@ def learn_once_cross_entropy( # Compute loss (Cross Entropy) # https://arize.com/blog-course/binary-cross-entropy-log-loss/ loss = -np.mean( - one_hot_targets * np.log(predictions) - + (1 - one_hot_targets) * np.log(1 - predictions) + one_hot_targets * np.log(predictions + EPSILON) + + (1 - one_hot_targets) * np.log(1 - predictions + EPSILON) ) # Backward pass @@ -120,11 +136,11 @@ def learn_once_cross_entropy( dC_dz2 = a2 - one_hot_targets dC_dw2 = np.matmul(a1.T, dC_dz2) - dC_db2 = np.sum(dC_dz2, axis=0, keepdims=True) + dC_db2 = np.sum(dC_dz2, axis=0) dC_da1 = np.matmul(dC_dz2, w2.T) dC_dz1 = dC_da1 * a1 * (1 - a1) dC_dw1 = np.matmul(a0.T, dC_dz1) - dC_db1 = np.sum(dC_dz1, axis=0, keepdims=True) + dC_db1 = np.sum(dC_dz1, axis=0) # Update weights and biases w1 -= learning_rate * dC_dw1 @@ -204,7 +220,7 @@ def test_mlp( z1 = np.matmul(a0, w1) + b1 a1 = 1 / (1 + np.exp(-z1)) z2 = np.matmul(a1, w2) + b2 - a2 = np.exp(z2) / np.sum(np.exp(z2), axis=1, keepdims=True) + a2 = softmax(z2) predictions = a2 # Compute accuracy @@ -237,7 +253,7 @@ def run_mlp_training( """ d_in = data_train.shape[1] - d_out = labels_train.shape[0] + d_out = max(labels_train) + 1 # Random initialization of the network weights and biaises w1 = 2 * np.random.rand(d_in, d_h) - 1 # first layer weights diff --git a/results/mlp.png b/results/mlp.png new file mode 100644 index 0000000000000000000000000000000000000000..7adf3430932d16a46e17db300e3a1335a7211aac Binary files /dev/null and b/results/mlp.png differ