Assignment3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment3.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "AHQ0j1oGxWbL"
      },
      "source": [
        "import pandas as pd\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torchsummary import summary\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\") \n",
        "from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "shI1xBwSQqdp",
        "outputId": "7e5067cb-8755-45af-c15e-f9372c1c402c"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NHg2yCDWxoIh"
      },
      "source": [
        "df = pd.read_csv('/content/drive/MyDrive/Dataset 2/Dataset 2.csv', encoding='latin1')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tpmd2bv-x-H_"
      },
      "source": [
        "class Sequences(Dataset):\n",
        "    def __init__(self, data):\n",
        "        self.vectorizer = CountVectorizer(stop_words='english')\n",
        "        self.sequences = self.vectorizer.fit_transform(data.text.tolist())\n",
        "        self.labels = data.polarity.tolist()\n",
        "        self.token2idx = self.vectorizer.vocabulary_\n",
        "        self.idx2token = {idx: token for token, idx in self.token2idx.items()}\n",
        "    \n",
        "    def __getitem__(self, i):\n",
        "        return self.sequences[i, :].toarray(), self.labels[i]\n",
        "    \n",
        "    def __len__(self):\n",
        "        return self.sequences.shape[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4uS7Pf9hyBu4"
      },
      "source": [
        "batch_size = 100\n",
        "\n",
        "dataset = Sequences(df)\n",
        "train_size = int(len(dataset) * .8);\n",
        "test_size = int(len(dataset) - train_size);\n",
        "train_dataset, test_dataset = torch.utils.data.random_split(dataset,[train_size, test_size])\n",
        "train_loader = DataLoader(train_dataset, batch_size=batch_size)\n",
        "test_loader = DataLoader(test_dataset, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rx34qgH5yHK3"
      },
      "source": [
        "input_size = len(dataset.token2idx)\n",
        "hidden_size = 128\n",
        "num_classes = 2\n",
        "num_layers = 3\n",
        "learning_rate = 0.01\n",
        "sequence_length = 5\n",
        "num_epochs = 5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v9ekxGPkyK13",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ecb6cf1c-bdb0-4c9b-84e8-5a4d191e60ea"
      },
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cpu')"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LIIYKSnqyORY"
      },
      "source": [
        "class LSTM(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
        "        super(LSTM, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.num_layers = num_layers\n",
        "        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n",
        "        self.fc = nn.Linear(self.hidden_size, num_classes)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        # Set initial hidden and cell states \n",
        "        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device) \n",
        "        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
        "        \n",
        "        # Forward propagate LSTM\n",
        "        out, _ = self.lstm(x, (h0, c0))\n",
        "        \n",
        "        # Decode the hidden state of the last time step\n",
        "        out = self.fc(out[:, -1, :])\n",
        "        return out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uKNon4C5yQ-n"
      },
      "source": [
        "model = LSTM(input_size, hidden_size, num_layers, num_classes).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MpuoC3MVySmI"
      },
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kZWpeUrQyZe7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "957448d4-8069-4dc3-e185-dd00466099fa"
      },
      "source": [
        "# Train the model\n",
        "y_pred = []\n",
        "y_test = []\n",
        "iter = []\n",
        "losses = []\n",
        "itr = 0\n",
        "total_step = len(train_loader)\n",
        "for epoch in range(num_epochs):\n",
        "    for i, (text, labels) in enumerate(train_loader):\n",
        "        labels = labels.to(device)\n",
        "        text = torch.tensor(text, dtype = torch.float)\n",
        "        # Forward pass\n",
        "        outputs = model(text)\n",
        "        loss = criterion(outputs, labels)\n",
        "        \n",
        "        # Backward and optimize\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        itr = itr + 1\n",
        "        if (i+1) % 20 == 0:\n",
        "            iter.append(itr)\n",
        "            losses.append(loss.item())\n",
        "            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'\n",
        "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
        "        \n",
        "# Test the model\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for text, labels in test_loader:\n",
        "        labels = labels.to(device)\n",
        "        text = torch.tensor(text, dtype = torch.float)\n",
        "        outputs = model(text)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "\n",
        "        y_pred.extend(predicted)\n",
        "        y_test.extend(labels)\n",
        "\n",
        "    print('Test Accuracy of the model on the 10000 test texts: {} %'.format(100 * correct / total)) \n",
        "\n",
        "# Save the model checkpoint\n",
        "# torch.save(model.state_dict(), 'model.ckpt')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [1/5], Step [20/83], Loss: 0.0354\n",
            "Epoch [1/5], Step [40/83], Loss: 0.0120\n",
            "Epoch [1/5], Step [60/83], Loss: 0.0169\n",
            "Epoch [1/5], Step [80/83], Loss: 0.0302\n",
            "Epoch [2/5], Step [20/83], Loss: 0.0010\n",
            "Epoch [2/5], Step [40/83], Loss: 0.0007\n",
            "Epoch [2/5], Step [60/83], Loss: 0.0004\n",
            "Epoch [2/5], Step [80/83], Loss: 0.0009\n",
            "Epoch [3/5], Step [20/83], Loss: 0.0003\n",
            "Epoch [3/5], Step [40/83], Loss: 0.0003\n",
            "Epoch [3/5], Step [60/83], Loss: 0.0003\n",
            "Epoch [3/5], Step [80/83], Loss: 0.0002\n",
            "Epoch [4/5], Step [20/83], Loss: 0.0003\n",
            "Epoch [4/5], Step [40/83], Loss: 0.0003\n",
            "Epoch [4/5], Step [60/83], Loss: 0.0002\n",
            "Epoch [4/5], Step [80/83], Loss: 0.0001\n",
            "Epoch [5/5], Step [20/83], Loss: 0.0003\n",
            "Epoch [5/5], Step [40/83], Loss: 0.0003\n",
            "Epoch [5/5], Step [60/83], Loss: 0.0003\n",
            "Epoch [5/5], Step [80/83], Loss: 0.0001\n",
            "Test Accuracy of the model on the 10000 test texts: 99.12748424624334 %\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8TVvVS9bycg8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "outputId": "93c815dd-ca30-4883-d10a-9fdb1ffd2be0"
      },
      "source": [
        "plt.plot(iter, losses, label='Loss')\n",
        "plt.title('Loss Curve')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Iteration')\n",
        "plt.legend()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f21d6d68590>"
            ]
          },
          "metadata": {},
          "execution_count": 52
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5gc1Xnn8e/b3TPdI2mmR+g2N4GEEcECBLZlYi84GBM7YBPLJHgtQmJ2w7OEENbrsNlEXseszWN2YZNd2zhsEhKwMQ4GL1k2chCR40CMLwEjMJKQAFvcVj0S6Do3jebS0+/+UdWjVqtH0z3TNd0j/T7P089Unaqufqv0qN8+deqcY+6OiIhIuWK1DkBERGYXJQ4REamIEoeIiFREiUNERCqixCEiIhVR4hARkYoocYiISEWUOOSkZmavm9kv1+izLzCzDWbWY2YHzOwnZvZvaxGLSCWUOERqwMzeCzwOfB84A1gA/C5w+RSPF69edCLHp8QhUoKZJc3sy2a2K3x92cyS4baFZvb3BTWFH5hZLNz2R2bWbWb9ZvaymV06wUf8CXCfu9/h7vs88Ky7/+vwOP/GzH5YFJOb2Rnh8tfN7M/DGssh4A/M7M3CBGJmV5rZlnA5ZmbrzOwVM9tvZt82s1OqfuHkpKDEIVLaZ4H3AOcD5wEXAH8cbvuPQAZYBCwB/jPgZvYLwE3Au929GfgV4PXiA5vZHOC9wMPTjPE3gNuAZuArwCHgA0XbHwiX/z3wMeBioAM4CNw1zc+Xk5QSh0hp1wC3uvsed98LfAH4rXDbKNAOnObuo+7+Aw8GfRsDksBKM2tw99fd/ZUSx55P8H9v9zRj/Dt3/5G759x9CPgWcDWAmTUDHw7LAG4APuvuGXcfBj4PXGVmiWnGICchJQ6R0jqANwrW3wjLILjNtAP4rpm9ambrANx9B/Bpgi/lPWb2oJl1cKyDQI4g+UzHzqL1B4BfC2+p/RrwnLvnz+E04JHw9loP8CJBolsyzRjkJKTEIVLaLoIv27xTwzLcvd/d/6O7nw58FLg535bh7g+4+0Xhex24o/jA7j4I/Avw68f5/EPAnPyKmbWV2Oeooa3dfTtBgruco29TQZBkLnf31oJXyt27jxODSElKHCLQYGapgleC4BbPH5vZIjNbCNwCfBPAzK4wszPMzIBegl/uOTP7BTP7QPiLfwg4TFCzKOUPgX9jZv/JzBaExz3PzB4Mt28Gzjaz880sRVCLKccDwH8Afgn43wXlfwHcZmanhZ+1yMzWlHlMkaMocYjABoIv+fzr88AXgU3AFmAr8FxYBrAC+B4wQFBz+F/u/gRB+8btwD7gTWAx8JlSH+juPyZoyP4A8KqZHQDuDmPB3X8G3Bp+zs+BH5Y6TgnfImgAf9zd9xWUfwVYT3B7rR94CvjFMo8pchTTRE4iIlIJ1ThERKQiShwiIlIRJQ4REamIEoeIiFTkpOg1unDhQl+2bFmtwxARmVWeffbZfe6+qLj8pEgcy5YtY9OmTbUOQ0RkVjGzN0qV61aViIhURIlDREQqosQhIiIVOSnaOEREpmp0dJRMJsPQ0FCtQ4lMKpWiq6uLhoaGsvZX4hAROY5MJkNzczPLli0jGNfyxOLu7N+/n0wmw/Lly8t6j25ViYgcx9DQEAsWLDghkwaAmbFgwYKKalRKHCIikzhRk0ZepecXaeIws8vM7GUz25GfJa1oe9LMHgq3P21my8LyC8zs+fC12cyuLHjP62a2NdwWaeeM+378Ot/ZvCvKjxARmXUiSxxmFgfuIpiNbCVwtZmtLNrtOuCgu58BfIkjs6W9AKx29/OBy4C/LJob+RJ3P9/dV0cVP8BDz+zkkZ9qgjQRqa158+bVOoSjRFnjuADY4e6vuvsI8CBQPOPYGuC+cPlh4FIzM3cfdPdsWJ6iaIrMmdLR2sSunsO1+GgRkboVZeLoJJjnOC8TlpXcJ0wUvUB+Gs1fNLNtBLOv3VCQSJxgFrNnzez6iT7czK43s01mtmnv3r1TOoGu+U10K3GISB16/vnnec973sOqVau48sorOXjwIAB33nknK1euZNWqVaxduxaA73//+5x//vmcf/75vOMd76C/v39an123j+O6+9MEcy6/HbjPzB5z9yHgInfvNrPFwD+a2Uvu/mSJ999NMBUnq1evnlKNpaM1Rf9Qlr6hUVpS5T3fLCInri98Zxvbd/VV9ZgrO1r4L796dsXv++QnP8lXv/pVLr74Ym655Ra+8IUv8OUvf5nbb7+d1157jWQySU9PDwB/+qd/yl133cWFF17IwMAAqVRqWjFHWePoBpYWrHeFZSX3Cdsw0sD+wh3c/UWCuZ3PCde7w797gEcIbolFoqO1CUC3q0SkrvT29tLT08PFF18MwLXXXsuTTwa/n1etWsU111zDN7/5TRKJoG5w4YUXcvPNN3PnnXfS09MzXj5VUdY4ngFWmNlyggSxFviNon3WA9cC/wJcBTzu7h6+Z6e7Z83sNOAs4HUzmwvE3L0/XP4QcGtUJ9BZkDjOamuJ6mNEZJaYSs1gpj366KM8+eSTfOc73+G2225j69atrFu3jo985CNs2LCBCy+8kI0bN3LWWWdN+TMiSxzhl/5NwEYgDtzr7tvM7FZgk7uvB+4B7jezHcABguQCcBGwzsxGgRxwo7vvM7PTgUfCZ44TwAPu/g9RnUM+cXQfVI1DROpHOp1m/vz5/OAHP+B973sf999/PxdffDG5XI6dO3dyySWXcNFFF/Hggw8yMDDA/v37Offcczn33HN55plneOmll+ozcQC4+wZgQ1HZLQXLQ8DHS7zvfuD+EuWvAudVP9LSFs5L0hiP0d1z4o5RIyL1b3BwkK6urvH1m2++mfvuu48bbriBwcFBTj/9dL72ta8xNjbGb/7mb9Lb24u786lPfYrW1lY+97nP8cQTTxCLxTj77LO5/PLLpxVP3TaO14NYzGhvTamNQ0RqKpfLlSx/6qmnjin74Q9/eEzZV7/61arGoyFHJtGR1iO5IiKFlDgmoU6AIiJHU+KYROf8Jt7qG2J0rHRVUUROfO41GbxixlR6fkock+hsTZFzeLNXDeQiJ6NUKsX+/ftP2OSRn4+jkk6BahyfRGEnwKWnzKlxNCIy07q6ushkMkx16KLZID8DYLmUOCYxnjh61c4hcjJqaGgoe2a8k4VuVU1CnQBFRI6mxDGJVEOcBXMb1QlQRCSkxFEGPZIrInKEEkcZOpU4RETGKXGUoaM16D1+oj6OJyJSCSWOMnS0phgcGaP38GitQxERqTkljjJ0zQ+frNLtKhERJY5ydOiRXBGRcUocZdAUsiIiRyhxlGHB3EaSiRi7NF6ViIgSRznMjM7WJt2qEhFBiaNs+UdyRUROdpEmDjO7zMxeNrMdZrauxPakmT0Ubn/azJaF5ReY2fPha7OZXVnuMaOiToAiIoHIEoeZxYG7gMuBlcDVZrayaLfrgIPufgbwJeCOsPwFYLW7nw9cBvylmSXKPGYkOlqb2NM/zHB2bCY+blr29g/z219/hr39w7UORUROQFHWOC4Adrj7q+4+AjwIrCnaZw1wX7j8MHCpmZm7D7p7NixPAfku2+UcMxIdrcEkJ7NhQqcnXtrD4y/t4dk3DtQ6FBE5AUWZODqBnQXrmbCs5D5hougFFgCY2S+a2TZgK3BDuL2cYxK+/3oz22Rmm6oxAUvnLOoEuDnTA8yOJCcis0/dNo67+9PufjbwbuAzZlb+vIbB++9299XuvnrRokXTjmc2zcuxJdMLwO4+JQ4Rqb4oE0c3sLRgvSssK7mPmSWANLC/cAd3fxEYAM4p85iRaEsHeWtXnc/LMTQ6xou7+wDVOEQkGlEmjmeAFWa23MwagbXA+qJ91gPXhstXAY+7u4fvSQCY2WnAWcDrZR4zEslEnMXNybp/surF3X1kc44Z7FbiEJEIRDbnuLtnzewmYCMQB+51921mdiuwyd3XA/cA95vZDuAAQSIAuAhYZ2ajQA640d33AZQ6ZlTnUGw29OXI36Z692mnqMYhIpGILHEAuPsGYENR2S0Fy0PAx0u8737g/nKPOVM6W5vGbwPVq82ZHhbOS/KOU1v52o9ex90xs1qHJSInkLptHK9HHa2pup/QaUuml/O60rSlU4yM5ThwaKTWIYnICUaJowKdrU0MZ3Psr9Mv44HhLK/sHWBVVyvtYWP+m3qySkSqTImjAvU+vPrWTC/usGppmrZ0EKvaOUSk2pQ4KlDviSPf8e+8ghqHnqwSkWqLtHH8RJOfQjZTp50At2R6WHpKE6fMbWQs58RjphqHiFSdahwVSDc1MKcxXredADfv7GVVVysA8ZixpDmpGoeIVJ0SRwXMjI46HV59/8Aw3T2HOa8rPV62JJ3izb76i1VEZjcljgp1tjaxq7f+vozzHf/yNQ6A9nRKt6pEpOqUOCrUUadTyG7O9GAG53QeqXG0tTSxu3eorvudiMjso8RRoc7WFPsPjTA0Wl8TOm3e2cMZi+YxL3nkeYf2dIrBkTH6h7PHeaeISGWUOCqUn5ejnto53J0tmd6jblPBkRF9dbtKRKpJiaNCHen6m9Cpu+cw+w+NcP7S9FHl6sshIlFQ4qhQPXYCLNUwDrCkJV/jqJ9YRWT2U+KoUFs6Rcygu476cmzO9NAQN85qbz6qPJ84VOMQkWpS4qhQQzzGkpZUXT1ZtWVnL29vbyGZiB9V3piIsXBekrc00KGIVJESxxTUUyfAXM7Z2t3Lqq50ye3t6ZRqHCJSVUocU1BPnQBf3TfAwHD2mPaNvDZ1AhSRKlPimIKO1iZ29wyRy9W+Y93mnUHD+PlLSycO1ThEpNqUOKagszWYXW/fwHCtQ2FLpoc5jXHetmheye1t6RS9h0cZHFEnQBGpjkgTh5ldZmYvm9kOM1tXYnvSzB4Ktz9tZsvC8g+a2bNmtjX8+4GC9/xzeMznw9fiKM+hlHwnwHroy7E508s5nWnisdLzire1qBOgiFRXZInDzOLAXcDlwErgajNbWbTbdcBBdz8D+BJwR1i+D/hVdz8XuBa4v+h917j7+eFrT1TnMJF8X45aJ46RbI7tu/uOGhG3WJumkBWRKouyxnEBsMPdX3X3EeBBYE3RPmuA+8Llh4FLzczc/afuviss3wY0mVkywlgrUi+dAH/2Vj8j2dyEDeMA7ZpCVkSqLMrE0QnsLFjPhGUl93H3LNALLCja59eB59y9sEHha+Ftqs+ZWcl7NGZ2vZltMrNNe/func55HKMl1UBzMlHzCZ2e33lkqtiJtKkToIhUWV03jpvZ2QS3r36noPia8BbW+8LXb5V6r7vf7e6r3X31okWLqh5b5/ymmk8huyXTw/w5DSw9pWnCfZoa47TOaVCNQ0SqJsrE0Q0sLVjvCstK7mNmCSAN7A/Xu4BHgE+6+yv5N7h7d/i3H3iA4JbYjKuHToD5EXEnqHSNa2vRI7kiUj1RJo5ngBVmttzMGoG1wPqifdYTNH4DXAU87u5uZq3Ao8A6d/9RfmczS5jZwnC5AbgCeCHCc5hQR2uqpp0AB0ey/Oyt/uM2jOe1aQpZEamiyBJH2GZxE7AReBH4trtvM7Nbzeyj4W73AAvMbAdwM5B/ZPcm4AzglqLHbpPARjPbAjxPUGP5q6jO4Xg6W+fQMzjKoRpNkrRtVx85P3ZE3FI0hayIVFNi8l2mzt03ABuKym4pWB4CPl7ifV8EvjjBYd9VzRinqqM1aHTe1XOYFUuaJ9m7+jaHDeOrlpZR42hpYt/ACCPZHI2Jum7WEpFZQN8iU9RZ474cWzK9tKdTLG5OTbpvfkInjZIrItWgxDFFte49vjnTM+GIuMXUCVBEqkmJY4oWN6eIx6wmT1b1DI7wxv5BzptgYMNimkJWRKpJiWOK4jGjrSVVk06A+alij9fxr9CStKaQFZHqUeKYhs75TTWZCXBLJmgYP6ezvFtVzckEcxvjqnGISFUocUxDZ2tTTdo4Nmd6OX3hXNJNDWXtb2aa0ElEqkaJYxo6WlO82TfE2AxP6LSlgobxvPZ0kxrHRaQqlDimobN1DmM5Z0//zH0hv9k7xFt9w2V1/CukGoeIVIsSxzTkOwHOZDvH5rB9o9wnqvLa0yn29A+THctFEZaInESUOKahFp0At2R6SMSMsztaKnpfWzrFWM7ZNzASUWQicrJQ4piGIxM6zdwtoC2ZXs5c0kyqIV7R+47My6FHckVkepQ4pmFuMkHrnAa6ewZn5PPcnS2ZXs4rY3yqYuO9x9XOISLTpMQxTR3pphmrcbyxf5Dew6MVN4xDwRSyerJKRKZJiWOaZnJCp3zDeKWP4gLMn9NAYyKmGoeITJsSxzR1zWDv8c07e0kmYpw5hWHczYz2tGYCFJHpU+KYpo7WFP3DWfqGRiP/rC2ZHs7pTNMQn9o/W1uL+nKIyPQpcUzTkSeroq11ZMdyvLCrd0q3qfLa0il2awpZEZkmJY5pGu/LEfHtqp/vGWBoNFf2iLiltKVTvNU7TG6Gh0gRkROLEsc0dc5QjWPLNBrG89pbUoyM5TgwqE6AIjJ1kSYOM7vMzF42sx1mtq7E9qSZPRRuf9rMloXlHzSzZ81sa/j3AwXveVdYvsPM7jQzi/IcJrNwXpLGeIzuiB/J3ZzppTmVYNmCuVM+Rlv+kVy1c4jINESWOMwsDtwFXA6sBK42s5VFu10HHHT3M4AvAXeE5fuAX3X3c4FrgfsL3vPnwL8DVoSvy6I6h3LEYkZ7ayryYUc27wxGxI3Fpp4n29UJUESqIMoaxwXADnd/1d1HgAeBNUX7rAHuC5cfBi41M3P3n7r7rrB8G9AU1k7agRZ3f8rdHfgG8LEIz6EsQSfA6BLH0OgYL7/ZP632DSiYQladAEVkGqJMHJ3AzoL1TFhWch93zwK9wIKifX4deM7dh8P9M5McEwAzu97MNpnZpr179075JMoRdSfA7bv7yOZ8Sj3GCy2YlyQRM00hKyLTUteN42Z2NsHtq9+p9L3ufre7r3b31YsWLap+cAU65zfxVt8QoxENWb5lZ34o9ak3jEMwT/ri5qQ6AYrItESZOLqBpQXrXWFZyX3MLAGkgf3hehfwCPBJd3+lYP+uSY454zpbU+Q8uraDLZleFjUnx0e4nQ5N6CQi0xVl4ngGWGFmy82sEVgLrC/aZz1B4zfAVcDj7u5m1go8Cqxz9x/ld3b33UCfmb0nfJrqk8DfRXgOZYm6E+DmTA/ndaWpxgNkmkJWRKYrssQRtlncBGwEXgS+7e7bzOxWM/touNs9wAIz2wHcDOQf2b0JOAO4xcyeD1+Lw203An8N7ABeAR6L6hzKFeWETn1Do7y679C02zfy8jWO4NkCEZHKJcrZyczmAofdPWdmZwJnAY+5+3EHaHL3DcCGorJbCpaHgI+XeN8XgS9OcMxNwDnlxD1ToqxxvJDpxX16Hf8KtadTDI6M0TeUJd3UUJVjisjJpdwax5NAysw6ge8CvwV8PaqgZptUQ5wFcxsj6QS4OdMLMO1HcfM0oZOITFe5icPcfRD4NeB/ufvHgbOjC2v26ZzfFMmtqi2ZHk49ZQ7z5zZW5XiaQlZEpqvsxGFm7wWuIWi0Bqhs0usTXFSdALdkpjcibjHVOERkuspNHJ8GPgM8EjZwnw48EV1Ys0++E2A1G533DQzT3XO4arepABY3pzBDfTlEZMrKahx39+8D3wcwsxiwz90/FWVgs03n/CYGR8boPTxK65zq3Faqxoi4xRoTMRbOS/KWHskVkSkqq8ZhZg+YWUv4dNULwHYz+0/Rhja7dLYGt4AyVZyXY/POXmIG53RWL3EAmkJWRKal3FtVK929j2BAwceA5QRPVkkoikdyN2d6WLG4mbnJsiqGZdMUsiIyHeUmjgYzayBIHOvD/hvqQVag2onD3aveMJ7Xlk7pqSoRmbJyE8dfAq8Dc4Enzew0oC+qoGajBXMbSSZiVXskN3PwMAcOjbBqafUaxvPa0in6hrIcGs5W/dgicuIrK3G4+53u3unuH/bAG8AlEcc2q5gZna1N7KpSJ8At4x3/ql/jGJ/QSQ3kIjIF5TaOp83sf+bntzCz/0FQ+5ACHa3V6wS4JdNDYzzGWW0tVTleobaW4LbaW2rnEJEpKPdW1b1AP/Cvw1cf8LWogpqtOquUONydH7+yn7e3N9OYqP44lOMzASpxiMgUlPu4ztvc/dcL1r9gZs9HEdBs1tHaxN7+YYazYyQTU+9Y/w8vvMnW7l7+65XnVjG6I9p0q0pEpqHcn7OHzeyi/IqZXQjosZwiHa3TH85jODvGf3vsJc5qa+YT7146+RumINUQZ/6cBj1ZJSJTUm6N4wbgG2aWb6k9yJEJmCTUOT+cl+PgYU5bMLUmoG/8+A3+34FBvvHbFxCPTX/ipoksUV8OEZmicocc2QycZ2Yt4XqfmX0a2BJlcLPNdCd0OnBohDsf/znv/4VF/NKZ0c6Trt7jIjJVFbW8untf2IMcghn7pEC+7WCqj+Te+U8/Z3BkjM9++O3VDKuktnSTahwiMiXTeWQnuvsos1QyEWdxc5LunsGK37tjzwD3P/UGV1+wlBVLmiOI7mjt6RT7D40wnB2L/LNE5MQyncShIUdK6JhiJ8DbH3uROQ1xPv3LZ0YQ1bHytaM9fcMz8nkicuI4buIws34z6yvx6gc6Jju4mV1mZi+b2Q4zW1die9LMHgq3P21my8LyBWb2hJkNmNmfFb3nn8NjPh++Fld0xhHrbK18Qqcf79jH917cw42XnMHCecmIIjua+nKIyFQdt3Hc3ad8z8TM4sBdwAeBDPCMma139+0Fu10HHHT3M8xsLXAH8AlgCPgccE74KnaNu2+aamxR6pzfxPdefAt3x2zyu3ljOeeLj75IZ2sT//bCZdEHGNIUsiIyVdXvlnzEBcAOd3/V3UeAB4E1RfusAe4Llx8GLjUzc/dD7v5DggQyq3SkUwxnc+w/NFLW/n/7XIbtu/tYd/lZpBpmbjZeTSErIlMVZeLoBHYWrGfCspL7uHsW6AUWlHHsr4W3qT5nE/ysN7Pr82Nr7d27t/Lop6iS4dUPDWf5040v845TW7liVXvUoR2lOdXAvGRCt6pEpGJRJo6oXOPu5wLvC18lJ5Ry97vdfbW7r160KNo+EYUKOwFO5i+ffJU9/cP88UdWlnVbq9ra0ilNISsiFYsycXQDhWNmdIVlJfcxswSQBvYf76Du3h3+7QceILglVjfK7QT4Zu8Qdz/5Clesauddp82fidCOoU6AIjIVUSaOZ4AVZrbczBqBtcD6on3Wc2TokquAx919wsd8zSxhZgvD5QbgCoI50OtGuqmBOY3xSR/J/ZONL5Nz+KPLzpqhyI6lKWRFZCqqO5l1AXfPmtlNwEYgDtzr7tvM7FZgk7uvB+4B7jezHcABguQCgJm9DrQAjWb2MeBDwBvAxjBpxIHvAX8V1TlMRX5Cp+N1Atya6eVvn8tww8VvY+kpc2YwuqO1p1Ps6R8iO5YjEZ+Ndy1FpBYiSxwA7r4B2FBUdkvB8hDw8Qneu2yCw76rWvFF5XidAN2dLz66nVPmNnLjJW+b4ciOtiSdIuewd2CY9nRTTWMRkdlDPzMj0HGcToDf3f4WT792gN//4Jm0pBpmOLKjqROgiEyFEkcEOluDcaCGRo8eB2okm+P2x17ijMXzuDqiuTYqkZ9CVu0cIlIJJY4IjD+SW1Tr+OZTb/DavkN89iNvr4s2hXZ1AhSRKaj9t9cJqCN9bCfAnsERvvJPP+d9Kxby/ojn2ihX65wGkomYppAVkYoocUSgVO/xO/9pB/1Do3z2I2+vSWe/UsxMfTlEpGJKHBFoS6eI2ZHe46/tO8Q3/uV1PvHupZzV1lLb4IoEU8hqoEMRKZ8SRwQa4jGWtKToDh/Jvf2xF0kmYvz+B2dmro1KqMYhIpVS4ohI/pHcp17dz8Ztb3HjJWewuDlV67CO0ZZu4q2+IXI5zcslIuVR4ohIZ2sTOw8O8sVHt9ORTnHdRctrHVJJ7ekUo2POgcHyhoEXEVHiiEhHaxOZg4d5obuPP7xsZufaqITm5RCRSilxRKSzNfhCPq8rzUfPm3SW3ZpR73ERqZQSR0RWdqRpiBufu2IlsVh9PH5bSn4KWT1ZJSLlinSQw5PZu06bz9bP/0rd3qLKWzAvSSJmqnGISNlU44hQvScNgHjMwr4cShwiUh4lDqFNfTlEpAJKHKK5x0WkIkocQntLUOM4zqy9IiLjlDiEtnSKw6Nj9B3O1joUEZkFlDhkvBPg7j49kisik4s0cZjZZWb2spntMLN1JbYnzeyhcPvTZrYsLF9gZk+Y2YCZ/VnRe95lZlvD99xp9TJG+SymToAiUonIEoeZxYG7gMuBlcDVZrayaLfrgIPufgbwJeCOsHwI+BzwByUO/efAvwNWhK/Lqh/9yaUtrSlkRaR8UdY4LgB2uPur7j4CPAisKdpnDXBfuPwwcKmZmbsfcvcfEiSQcWbWDrS4+1MetOR+A/hYhOdwUljcnMRMiUNEyhNl4ugEdhasZ8Kykvu4exboBRZMcszMJMcEwMyuN7NNZrZp7969FYZ+cmmIx1g0L6nEISJlOWEbx939bndf7e6rFy2qjzm+61l7OsVu9eUQkTJEmTi6gaUF611hWcl9zCwBpIH9kxyza5JjyhRoClkRKVeUieMZYIWZLTezRmAtsL5on/XAteHyVcDjfpxeaO6+G+gzs/eET1N9Evi76od+8tEUsiJSrshGx3X3rJndBGwE4sC97r7NzG4FNrn7euAe4H4z2wEcIEguAJjZ60AL0GhmHwM+5O7bgRuBrwNNwGPhS6apLd1E/1CWgeEs85IaNFlEJhbpN4S7bwA2FJXdUrA8BHx8gvcum6B8E3BO9aIUONKX483eIc5YPK/G0YhIPTthG8elMvne4xrsUEQmo8QhgHqPi0j5lDgECJ6qAk0hKyKTU+IQIJitcP6cBtU4RGRSShwyri3dpN7jIjIpJQ4Zp74cIlIOJQ4ZpylkRaQcShwyrr0lxf5DIwyNjtU6FBGpY0ocMi7fl2NP33CNIxGReqbEIePGp5DVI7kichxKHDJufNgRtXOIyHEocci4/BSyerJKRGp5mwQAABCPSURBVI5HiUPGzUsmaE4m1JdDRI5LiUOO0pZOKXGIyHEpcchR2jSFrIhMQolDjtKmKWRFZBJKHHKU9nSKPf3DjI7lah2KiNQpJQ45Slu6CXfY269OgCJSmhKHHEUTOonIZCJNHGZ2mZm9bGY7zGxdie1JM3so3P60mS0r2PaZsPxlM/uVgvLXzWyrmT1vZpuijP9kpClkRWQyiagObGZx4C7gg0AGeMbM1rv79oLdrgMOuvsZZrYWuAP4hJmtBNYCZwMdwPfM7Ex3z4++d4m774sq9pOZahwiMpkoaxwXADvc/VV3HwEeBNYU7bMGuC9cfhi41MwsLH/Q3Yfd/TVgR3g8iVi6qYFUQ0xPVonIhKJMHJ3AzoL1TFhWch93zwK9wIJJ3uvAd83sWTO7PoK4T2pmRluLJnQSkYlFdqsqQhe5e7eZLQb+0cxecvcni3cKk8r1AKeeeupMxzirqfe4iBxPlDWObmBpwXpXWFZyHzNLAGlg//He6+75v3uAR5jgFpa73+3uq9199aJFi6Z9MieT9nSTahwiMqEoE8czwAozW25mjQSN3euL9lkPXBsuXwU87u4elq8Nn7paDqwAfmJmc82sGcDM5gIfAl6I8BxOSvkpZHM5r3UoIlKHIrtV5e5ZM7sJ2AjEgXvdfZuZ3Qpscvf1wD3A/Wa2AzhAkFwI9/s2sB3IAr/n7mNmtgR4JGg/JwE84O7/ENU5nKza0ymyOWf/oREWNSdrHY6I1JlI2zjcfQOwoajsloLlIeDjE7z3NuC2orJXgfOqH6kUamsJJ3TqHVLiEJFjqOe4HENTyIrI8ShxyDHaNIWsiByHEoccY+HcJImY6ckqESlJiUOOEYsZS1rUl0NESlPikJLa1QlQRCagxCEltaVTauMQkZKUOKSk9nSK3b2HCfpjiogcocQhJS1pSTE0mqP38GitQxGROqPEISW1p5sAzcshIsdS4pCSxvtyKHGISBElDilJMwGKyESUOKSkRc1JYqbe4yJyLCUOKakhHmNRc1JTyIrIMZQ4ZEKaQlZESlHikAlpClkRKUWJQybUnm5S4hCRY0Q6kZPMbm3pFP3DWW57dDuLm1MsbG5kwdwkC+clWTivkVPmNpKI19dvD3fnrb5htu3qpSEe49zONPPnNtY6LJETihKHTOg9py9g6SlN3PcvbzCSzZXcZ/6cBhbOS7JgXmOYUIKksiBcbmtJ0Tm/iflzGgin/K0ad+f/HRhk264+Xuju5YVdfWzf1cu+gZGj9jv1lDmc25VmVWeaVV2tnNPZQnOqoaqxiJxM7GQYi2j16tW+adOmWocxa7k7A8NZ9g2MsG9gmP0Dw+wdGGH/wHC4HpTnt/cPZY85xpzGOJ2tTXTNb6JzfhNd8+cctb5oXvK4iSU7luPVfYfYtquXF7qDRLF9d9/4ZyVixoolzZzT0cI5nWnO7mhhJJtjS3cvWzI9bN7ZS3dP8ISYGZy+cC6rulpZ1ZVmVVeale1pmhrj0VxAkVnKzJ5199XHlEeZOMzsMuArQBz4a3e/vWh7EvgG8C5gP/AJd3893PYZ4DpgDPiUu28s55ilKHHMrKHRMQ4cCpLI7t4hMgcP033wMJmDg3T3HCZz8PAxY2A1JmJ0teaTShOdrU2kmxp4+a1+tu3q48XdfQyNBrWeZCLG29tbOKezhbM70pzTkebMtnkkE8f/4t8/MMyW7l62ZsJkkullb/8wAPGYsWLxPM7rauXcrjTndbXyC23NNCbq61bcbOPuVa9pysyZ8cRhZnHgZ8AHgQzwDHC1u28v2OdGYJW732Bma4Er3f0TZrYS+BZwAdABfA84M3zbcY9ZihJH/ekfGqW7J59QDhcsB8klf7upOZlgZUeYIDqD2sTpC+dWrW3lzd4htmR62JLpHa+d9AwGSa0xHuOs9mZa5zSSyzk5D185yLkz5k7Ogy/HsVywfNR+fuSL0wwMiIXLsfDLtHA9ZkD4N79vYyLGnMY4TY0J5jbGaWqMM6cxzpzGRPg32DanISxPBuVNDXGSiRhDozkGR7MMjowxODzG4EiWw6NjwfrIGIdHsgXLYwyOjjE4HJaNBtuHsznGco474XkeOb9ceO6lto3lgu+WxkSM5mSCuckE85IJ5qXCv2FZc6pgOdw+d3x7nIZ4jIZYjETcguXwbyJuNMRixGLlJaZczjk8Oha88uc8GlyToYJrUrx8eGSM0bEco2POWC5HNhecc+F6diwoyxatj+ZyuAc/TBIxK/obIx4uH1UeN+Kx2Ph6qiEWXI/GwusSXJt5yWPLJvsBVYmJEkeUbRwXADvc/dUwgAeBNUDhl/wa4PPh8sPAn1nw82QN8KC7DwOvmdmO8HiUcUyZBZpTDZzV1sBZbS0ltx8eGaPn8AhLmlNlfzFMRVs6RVu6jQ+d3QYEX/SZg4fZnOlha6aXrd299B4eJWYQNwu+4GOQsOA/vZkRt3wCMOKxMAnEgn0N8PC47uAEicfJJ5ZwGxyVbDz8Uh7J5ugZHGUw/ILPf7nnv5SrIRGzoxJSU5iEWlIJ2ltSJBtiwbnHgqR25LzDJGfBF1zMGD/vwm1D2TEODWcZGMoyMBy89vQP8ereLAPDYwwMj47XJqci/6XbGC9MLsFydszHk+VUPiPVEKOpIU5jIjb+RZ+I57/kgySW//xELEayITGeFBIxIx4PrkcuTCpBcgmTTJhchrNjjDlBEgrL8vtlx3IcHh3j0PAYI2Plxd8QtyCJNAYJ5f/+3oVVvw0bZeLoBHYWrGeAX5xoH3fPmlkvsCAsf6rovZ3h8mTHlBNAU2OcpsamGf9cM2PpKXNYesocrljVMeOfXw53Z2QsF9QgRo+uNQwWLI9kc6Qa8rWSOHPDWsp4kmhI0NQYr4vbcdmxHIeGx+gfHmVgOMuh4Sz9Q8G55H/tZ8dyR5Zzwd/RsCw75uPr2VyOkWywTyIWGz/npoJrESwnaGqM0RReh3xNLb+cSsQj/dFSqZFsLkjAw1kOjWQ5NBwk5PGy4SyHRsbGl/N/kxH8+56wT1WZ2fXA9QCnnnpqjaMRqR4zI5kIbknMr3UwVZKIx0jPiZGeo6fdJtKYiNGYaKyLx8uj/KnRDSwtWO8Ky0ruY2YJIE3QSD7Re8s5JgDufre7r3b31YsWLZrGaYiISKEoE8czwAozW25mjcBaYH3RPuuBa8Plq4DHPWitXw+sNbOkmS0HVgA/KfOYIiISochuVYVtFjcBGwkenb3X3beZ2a3AJndfD9wD3B82fh8gSASE+32boNE7C/yeu48BlDpmVOcgIiLHUgdAEREpaaLHcWv/OIWIiMwqShwiIlIRJQ4REamIEoeIiFTkpGgcN7O9wBu1jqOEhcC+WgcxAcU2NfUaW73GBYptqmYittPc/ZiOcCdF4qhXZrap1BML9UCxTU29xlavcYFim6paxqZbVSIiUhElDhERqYgSR23dXesAjkOxTU29xlavcYFim6qaxaY2DhERqYhqHCIiUhElDhERqYgSxwwys9fNbKuZPW9mm8KyU8zsH83s5+HfGZmbx8zuNbM9ZvZCQVnJWCxwp5ntMLMtZvbOGY7r82bWHV63583swwXbPhPG9bKZ/UpUcYWftdTMnjCz7Wa2zcz+Q1heD9dtothqfu3MLGVmPzGzzWFsXwjLl5vZ02EMD4VTJRBOp/BQWP60mS2b4bi+bmavFVyz88PyGfv3LIgxbmY/NbO/D9dres3GBfMb6zUTL+B1YGFR2X8H1oXL64A7ZiiWXwLeCbwwWSzAh4HHAAPeAzw9w3F9HviDEvuuBDYDSWA58AoQjzC2duCd4XIz8LMwhnq4bhPFVvNrF57/vHC5AXg6vB7fBtaG5X8B/G64fCPwF+HyWuChGY7r68BVJfafsX/Pgs+8GXgA+PtwvabXLP9SjaP21gD3hcv3AR+biQ919ycJ5kApJ5Y1wDc88BTQambtMxjXRNYAD7r7sLu/BuwALogirjC23e7+XLjcD7wIdFIf122i2CYyY9cuPP+BcLUhfDnwAeDhsLz4uuWv58PApWZW9cm/jxPXRGbs3xPAzLqAjwB/Ha4bNb5meUocM8uB75rZsxbMiQ6wxN13h8tvAktqE9pxY+kEdhbsl+H4X0pRuCm8PXBvwe28msUV3gp4B8Gv1Lq6bkWxQR1cu/CWy/PAHuAfCWo4Pe6eLfH547GF23uBBTMRl7vnr9lt4TX7kpkli+MqEXMUvgz8IZAL1xdQB9cMlDhm2kXu/k7gcuD3zOyXCjd6UM+si+ej6ykW4M+BtwHnA7uB/1HLYMxsHvC3wKfdva9wW62vW4nY6uLaufuYu58PdBHUbM6qRRzFiuMys3OAzxDE927gFOCPZjouM7sC2OPuz870Z5dDiWMGuXt3+HcP8AjBf6C38tXd8O+e2kU4YSzdwNKC/brCshnh7m+F/8FzwF9x5JbKjMdlZg0EX8x/4+7/Jyyui+tWKrZ6unZhPD3AE8B7CW715KevLvz88djC7Wlg/wzFdVl428/dfRj4GrW5ZhcCHzWz14EHCW5RfYU6uWZKHDPEzOaaWXN+GfgQ8AKwHrg23O1a4O9qEyEcJ5b1wCfDp0reA/QW3JqJXNF95CsJrls+rrXhEyXLgRXATyKMw4B7gBfd/X8WbKr5dZsotnq4dma2yMxaw+Um4IMEbTBPAFeFuxVft/z1vAp4PKzJzURcLxX8CDCCNoTCazYj/57u/hl373L3ZQSN3Y+7+zXU+JoVBqjXDLyA0wmeYtkMbAM+G5YvAP4J+DnwPeCUGYrnWwS3LkYJ7pVeN1EsBE+R3EVwX3orsHqG47o//NwtBP9B2gv2/2wY18vA5RFfs4sIbkNtAZ4PXx+uk+s2UWw1v3bAKuCnYQwvALcU/J/4CUHD/P8GkmF5KlzfEW4/fYbjejy8Zi8A3+TIk1cz9u9ZFOf7OfJUVU2vWf6lIUdERKQiulUlIiIVUeIQEZGKKHGIiEhFlDhERKQiShwiIlIRJQ6RCpjZQPh3mZn9RpWP/Z+L1n9czeOLVIsSh8jULAMqShwFPX4nclTicPd/VWFMIjNCiUNkam4H3hfO1/D74WB5f2Jmz4SD4/0OgJm938x+YGbrge1h2f8NB7rclh/s0sxuB5rC4/1NWJav3Vh47BcsmM/lEwXH/mcze9jMXjKzv4lyRFSRvMl+AYlIaesI5rm4AiBMAL3u/u5wNNUfmdl3w33fCZzjwfDlAL/t7gfCYS6eMbO/dfd1ZnaTBwPuFfs1gkEKzwMWhu95Mtz2DuBsYBfwI4Ixjn5Y/dMVOUI1DpHq+BDBOEbPEwxnvoBg/CeAnxQkDYBPmdlm4CmCgelWcHwXAd/yYLDCt4DvE4zcmj92xoNBDJ8nuIUmEinVOESqw4B/7+4bjyo0ez9wqGj9l4H3uvugmf0zwThDUzVcsDyG/k/LDFCNQ2Rq+gmmaM3bCPxuOLQ5ZnZmOApysTRwMEwaZxFMQZo3mn9/kR8AnwjbURYRTK8b2SjAIpPRrxORqdkCjIW3nL5OMFfCMuC5sIF6L6WnAf4H4AYze5FgVNqnCrbdDWwxs+c8GEI77xGC+Ss2E4yA+4fu/maYeERmnEbHFRGRiuhWlYiIVESJQ0REKqLEISIiFVHiEBGRiihxiIhIRZQ4RESkIkocIiJSkf8Pyko8f+eb6gAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xPx5ZGz6yf3v",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "outputId": "64cd6c80-e010-4c9f-c7c4-f5c195439223"
      },
      "source": [
        "def show_confusion_matrix(confusion_matrix):\n",
        "  hmap = sns.heatmap(confusion_matrix, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
        "  hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')\n",
        "  hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')\n",
        "  plt.ylabel('True polarity')\n",
        "  plt.xlabel('Predicted polarity');\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "df_cm = pd.DataFrame(cm, index=[0,1], columns=[0,1])\n",
        "show_confusion_matrix(df_cm)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAELCAYAAAAvPrYDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfx0lEQVR4nO3debxVZb3H8c/3gAamIsggAgom6stZwzmNNBWHG1phaiVOlyw17zWvs+KQOaQ5XEtDwDnnCdOrkoXYgAJqOJbHEVBCA+eB6Xf/WM/BDRzO2edw9t5n7f1991qvs/azpmcXfc9znvWs9SgiMDOzfKqrdAXMzKz1HOJmZjnmEDczyzGHuJlZjjnEzcxyzCFuZpZjDnEzsxUkaayk2ZKeW6r8WEkvSXpe0kUF5adIqpf0D0l7FpQPSWX1kk4u6toeJ25mtmIk7QJ8BNwQEZumsm8ApwH7RMTnknpGxGxJGwO3ANsCawN/ADZIp/onsDswA5gMHBQRLzR17Y6l+EJmZrUkIiZK6r9U8Y+BCyLi87TP7FQ+FLg1lb8mqZ4s0AHqI+JVAEm3pn3zE+KdtzrGfxbYEuZOvrLSVbB2qFNHtKLnaEnefPbMr38EjCgoGhURo5o5bANgZ0nnAZ8BJ0TEZKAPMKlgvxmpDGD6UuXbNVe3dhXiZmZlo+JvCabAbi60l9YR6AZsD2wD3C5pvRaeo6iLmJnVHq1wY745M4C7I7vx+KSkRUB3YCbQr2C/vqmMJsqXy6NTzKw21XUofmmde4FvAEjaAFgZeBcYBxwo6UuSBgADgSfJbmQOlDRA0srAgWnfJrklbma1qQXdKc2eSroFGAx0lzQDGAmMBcamYYfzgOGpVf68pNvJblguAI6OiIXpPMcADwMdgLER8Xxz13aIm1ltasPulIg4aDmbfrCc/c8Dzmuk/EHgwZZc2yFuZrWpDVvileQQN7PaVPobm2XhEDez2uSWuJlZjrV+1Em74hA3s9rk7hQzsxxzd4qZWY45xM3McqzO3SlmZvnlG5tmZjnm7hQzsxzz6BQzsxxzS9zMLMfcEjczyzG3xM3McsyjU8zMcszdKWZmOebuFDOzHKuSEK+Ob2Fm1lJS8Uuzp9JYSbPTfJpLb/uZpJDUPX2WpCsk1UuaJmnrgn2HS3o5LcOL+RoOcTOrTW072/11wJClCyX1A/YA3iwo3otshvuBwAjgqrRvN7IJlrcDtgVGSura7NcopnZmZlVHdcUvzYiIicCcRjZdCpwIREHZUOCGyEwC1pDUG9gTGB8RcyJiLjCeRn4xLM194mZWm0o8OkXSUGBmRPxdS16rDzC94POMVLa88iY5xM2sJqkFIS5pBFnXR4NRETGqif1XAU4l60opKYe4mdWkloR4CuzlhnYjvgIMABpa4X2BpyRtC8wE+hXs2zeVzQQGL1U+obkLuU/czGqTWrC0UEQ8GxE9I6J/RPQn6xrZOiJmAeOAQ9Iole2B9yPibeBhYA9JXdMNzT1SWZPcEjezmlRX13ZtWEm3kLWiu0uaAYyMiDHL2f1BYG+gHvgEOAwgIuZIOheYnPY7JyIau1m6BIe4mdWklnSnNCciDmpme/+C9QCOXs5+Y4GxLbm2Q9zMalJbhnglOcTNrDZVR4Y7xM2sNrklbmaWYw5xM7Mca8vRKZXkEDez2lQdDXGHuJnVJnenmJnlmEPczCzHHOJmZjmmOoe4mVluuSVuZpZjDnEzsxxziJuZ5Vl1ZLhD3Mxqk1viZmY55sfuzczyrDoa4qUNcUlDgMuBDsDoiLiglNdrz64e+X322mVT3pnzIYOG/WJx+Y8P/Do/OmBnFi4KHnr8OU67/D4ATjh8Dw4dugMLFy3iZxfdyR/+9iID1+3JjRcevvjYAX3W5NyrHuDK300o87excvvggw84+8zTqa//J5I4+9xfsMWWW1W6Wrnm7pRmSOoA/BrYnWyS0MmSxkXEC6W6Znt24/2TuPq2xxh97iGLy3YZNJB9B2/Gtt+7gHnzF9Cj66oAbLTeWgzbc2u2/u559O7RhQevPobN9juHl9+YzfYHZr8H6+rEKw+fx7g//b0i38fK66Lzz2Onr+3MJZddwfx58/j0s88qXaXcq5YQL2Wn0LZAfUS8GhHzgFuBoSW8Xrv2l6deYc77nyxRNmLYzlx87XjmzV8AwDtzPwJg38Gbc8fDTzFv/gLeeOvfvDL9XbbZtP8Sx35j2w15bcY7vPn23LLU3yrnww8/ZOrUyez/ne8CsNLKK7P66qtXuFb5J6nopYhzjZU0W9JzBWW/lPSSpGmS7pG0RsG2UyTVS/qHpD0LyoeksnpJJxfzPUoZ4n2A6QWfZ6QyS9Zftyc7bfUVJt5wAo+MPo6vbrwOAH16dGHGrC/Ceebsuazds8sSxw7b86vc/tDUstbXKmPmjBl07dqNM087hQO+sx9nnXkan3zySfMHWpPaMsSB64AhS5WNBzaNiM2BfwKnpOtuDBwIbJKO+Y2kDgW9F3sBGwMHpX2bVPHbs5JGSJoiacqCd5+vdHXKqmOHOrp1+TK7HHIxp156LzdddHjzBwErdezAPl/fjLvHP13iGlp7sHDhAl568QWGHXgQt991L507d2bs6FGVrlbuqU5FL82JiInAnKXKHomIBenjJKBvWh8K3BoRn0fEa0A9Wc9Fq3ovShniM4F+BZ/7prIlRMSoiBgUEYM6dt+khNVpf2b+6z3uffQZAKY8/waLFgXdu67KzHfep+9aXRfv16dnV96a/f7iz3t+bWOeeWk6s+d8WPY6W/n16rUWvXqtxeabbwHA7nsM4aUXa/LWUptqSUu8sLGZlhEtvNzhwP+l9eX1UrSq96KUIT4ZGChpgKSVyf58GFfC6+XO/ROm8fVtNgBg/XV6svJKHXl37kc8MGEaw/bcmpVX6si6a6/J+uv0YPJzry8+7oAhg9yVUkO69+hBr7XW4vXXXgXgiUl/Y72vfKXCtco/qfilsLGZlqL/FJJ0GrAAuLkU36Nko1MiYoGkY4CHyYYYjo2I2uovKXD9+Yey81cH0n2NVal/6FzOvfpBrr/3b/z2rO8z5Y5TmTd/IUeeeSMAL746i7seeZqn7zqNBQsX8V8X3M6iRQHAKp1WZtftNuKYn99Sya9jZXbyqWdwykknMH/+fPr27cc5Pz+/0lXKvXKMTpF0KLAvsFtERCpuqpei2d6LZa7xxXkrr/NWx7Sfyli7MHfylZWugrVDnTqu+KM6G5z4UNF588+LhjR7PUn9gd9HxKbp8xDgV8DXI+Kdgv02AX5H1ge+NvAoMJDs8aN/AruRhfdk4ODmGr9+YtPMalJdG04KIekWYDDQXdIMYCTZaJQvAeNTq39SRBwVEc9Luh14gayb5eiIWJjO0+LeC4e4mdWktgzxiDiokeIxTex/HnBeI+UPAg+25NoOcTOrSVXywKZD3MxqU7U8du8QN7OaVCUZ7hA3s9rklriZWY615Y3NSnKIm1lNckvczCzHqiTDHeJmVpvcEjczy7EqyXCHuJnVJrfEzcxyzKNTzMxyrEoa4g5xM6tN7k4xM8uxKslwh7iZ1Sa3xM3Mcsw3Ns3McswtcTOzHKuSDKeuuR0kTZV0tKSu5aiQmVk5SCp6KeJcYyXNlvRcQVk3SeMlvZx+dk3lknSFpHpJ0yRtXXDM8LT/y5KGF/M9mg1x4HtkMzJPlnSrpD1VLX+HmFnNkopfinAdMGSpspOBRyNiINmM9ien8r3IZrcfCIwArsrqo25kEyxvB2wLjCym8dxsiEdEfUScBmwA/A4YC7wh6ex0UTOz3GnLlnhETATmLFU8FLg+rV8P7FdQfkNkJgFrSOoN7AmMj4g5ETEXGM+yvxiWUUxLHEmbA5cAvwTuAoYBHwB/LOZ4M7P2pkOdil4kjZA0pWAZUcQlekXE22l9FtArrfcBphfsNyOVLa+8Sc3e2JQ0FXgPGAOcHBGfp01PSNqpuePNzNqjlnQKR8QoYFRrrxURISlae3xTihmdMiwiXi0skDQgIl6LiG+XolJmZqVWhlt7/5LUOyLeTt0ls1P5TKBfwX59U9lMYPBS5ROau0gx3Sl3FllmZpYbdSp+aaVxQMMIk+HAfQXlh6RRKtsD76dul4eBPSR1TTc090hlTVpuS1zSRsAmQBdJhS3u1YFOLf02ZmbtSVu2xCXdQtaK7i5pBtkokwuA2yUdAbwBHJB2fxDYG6gHPgEOA4iIOZLOBSan/c6JiKVvli6jqe6UDYF9gTWA/ygo/xD4z6K+mZlZO1XXhiEeEQctZ9NujewbwNHLOc9YshGARVtuiEfEfcB9knaIiL+15KRmZu1dlbw6pcnulBMj4iLgYEnL/JaJiJ+WtGZmZiVULc8sNtWd8mL6OaUcFTEzK6cqyfAmu1Pul9QB2CwiTihjnczMSq4t+8Qrqclx4hGx0A/0mFk1qpIML+phn2ckjQPuAD5uKIyIu0tWKzOzEqulSSE6Af8Gdi0oC8Ahbma5VRPdKQARcVg5KmJmVk7VEeHFvQCrE3AE2dObi5/UjIjDS1gvM7OSqpYhhsW8O+VGYC2yd90+RvZSlg9LWSkzs1Irw7tTyqKYEF8/Is4APo6I64F9yGaeMDPLrbacFKKSirmxOT/9fE/SpmQvN+9ZuiqZmZVeLY1OGZVei3gG2SsUVwXOLGmtzMxKrEoyvKjRKaPT6mPAeqWtjplZebT3bpJiNfUCrOObOjAiftX21TEzK4/qiPCmW+Krla0WZmZlVvUP+0TE2eWsiJlZOVXLjc1mhxhK6ivpHkmz03KXpL7lqJyZWalIxS/tWTHjxK8lG5WydlruT2VmZrlVJxW9FEPSf0t6XtJzkm6R1EnSAElPSKqXdJukldO+X0qf69P2/q3+HkXs0yMiro2IBWm5DujR2guambUHbdkSl9QH+CkwKCI2BToABwIXApdGxPrAXLJXmJB+zk3ll6b9WqWYceL/lvQD4Jb0+SCytxq2ubmTryzFaS3HJr3S7GTfVoMGb9hthc9RgiGGHYHOkuYDqwBvk7399eC0/XrgLOAqYGhaB7gTuFKS0iTKLVJMS/xw4ACyJzVnAd8F/GZDM8u1uhYskkZImlKwjCg8V0TMBC4G3iQL7/eBqcB7EbEg7TYD6JPW+wDT07EL0v5rtuZ7FPOwzxvAt1pzcjOz9qpDC0anRMQoYNTytqen2ocCA4D3yCbRGbKCVSxKMaNT1pN0v6R30uiU+yT5yU0zy7U2fovhN4HXIuKdiJhPNmnOTsAakhoay32BmWl9JtAPIG3vQiu7qYvpTvkdcDvQm2x0yh180T9uZpZLbfwWwzeB7SWtouyA3YAXgD+RdUEDDAfuS+vj0mfS9j+2pj8cigvxVSLixoLRKTdRMDmEmVketWVLPCKeILtB+RTwLFm2jgJOAo6XVE/W5z0mHTIGWDOVHw+c3NrvUczolP+TdDJwK9ncmt8DHpTULVXewwfMLHfaenBKRIwERi5V/CqwbSP7fgYMa4vrFhPiB6SfP1qq/ECyUHf/uJnlTtW/O6VBRAwoR0XMzMqpQ3VkeFEtcTOzqlMzLXEzs2pUJRnuEDez2lQlb6It6mEfSfqBpDPT53UkLXO31cwsT9r6LYaVUsw48d8AO5C9+ArgQ+DXJauRmVkZdKgrfmnPiulO2S4itpb0NEBEzG14J66ZWV6pSmbZLCbE50vqQDYmHEk9gEUlrZWZWYlVS594MSF+BXAP0FPSeWTP+Z9e0lqZmZVYzYR4RNwsaSrZC10E7BcRL5a8ZmZmJVSCSSEqotkQl7QO8AnZ3JqLyyLizVJWzMyslGqmJQ48QNYfLrK3Fw4A/gFsUsJ6mZmVVEsmhWjPiulO2azws6StgZ+UrEZmZmVQJRne8ic2I+IpSduVojJmZuVSJV3iRfWJH1/wsQ7YGnirZDUyMyuDuhoaJ75awfoCsj7yu0pTHTOz8qiJlnh6yGe1iDihTPUxMyuLaukTX+5bASR1jIiFZDM2m5lVlQ51KnophqQ1JN0p6SVJL0raQVI3SeMlvZx+dk37StIVkuolTUsDRlqlqVe7PJl+PiNpnKQfSvp2w9LaC5qZtQcleIvh5cBDEbERsAXwItkEyI9GxEDgUb6YEHkvYGBaRgBXtfZ7FNMn3gn4N7ArX4wXD+Du1l7UzKzS2rJPXFIXYBfgUICImAfMkzQUGJx2ux6YAJwEDAVuiIgAJqVWfO+IeLul124qxHumkSnP8UV4N4iWXsjMrD1pyRtmJY0gazE3GBURowo+DwDeAa6VtAUwFTgO6FUQzLOAXmm9DzC94PgZqaxNQ7wDsCo0Og7HIW5mudaSd6ekwB7VxC4dyYZfHxsRT0i6nC+6ThrOEZLaPDubCvG3I+Kctr6gmVl70KFtxxjOAGZExBPp851kIf6vhm4SSb2B2Wn7TKBfwfF9U1mLNfUXRZUMwDEzW5ZasDQnImYB0yVtmIp2A14AxgHDU9lw4L60Pg44JI1S2R54vzX94dB0S3y31pzQzCwPSvCwz7HAzWnms1eBw8gayrdLOgJ4Azgg7fsgsDdQT/aW2MNae9HlhnhEzGntSc3M2ru2fp94RDwDDGpk0zIN4jQq5ei2uG6LX4BlZlYN2vn8x0VziJtZTaqZmX3MzKpRC57EbNcc4mZWk9ydYmaWY+5OMTPLseqIcIe4mdWoKmmIO8TNrDbV0vRsZmZVx6NTzMxyrEoy3CFuZrXJ3SlmZjnmlriZWY45xM3McqyNJ4WoGIe4mdUkuU/czCy/qqQhXroQlzQW2BeYHRGbluo6eXfm6acw8bEJdOu2Jnff93sAfnXxhTw24U+stNJK9O23Duf8/HxWX331CtfUymXRwoX84vjDWGPNHhxz5iXccMV5vFH/EhFBrz7rMPy40+nUeRXmvDOLay87l08/+pBFixax//CfsNmgHStd/dyolpZ4KV/kdR0wpITnrwpD9/s2V/129BJl2++wE3fd+3vuvOd+1l23P2Ou+W2FameV8Oj9t7NWv/6LPw878r8444obOfN/b6Jrj15MeOBOAB647ToG7bQbp19+A0f+z7nccvUvK1TjfKpT8Ut7VrIQj4iJgKd4a8ZXB23D6l26LFG2405fo2PH7I+kzbfYktn/mlWJqlkFzH13Ns9O+Qtf2/1bi8s6r/JlACKC+Z9/vrgfQIJPP/0YgE8/+Ygu3bqXv8I5phb8p6jzSR0kPS3p9+nzAElPSKqXdFuaexNJX0qf69P2/ivyParllbpV696772KnnXepdDWsTG4ffRnfOfQYVLfk/zWvu/zn/M8h+zBr5hvsuu8wAP7joCN5YsJDnHTYt7jy7J9x4IifVaLKuVWClvhxwIsFny8ELo2I9YG5wBGp/Ahgbiq/NO3X+u+xIgdbaV3z26vo0LED++z7reZ3ttybNvnPrNalK+uuv9Ey2w497nQuuu5+evftz+TH/wDAkxPHs+Ou+3DhteM4ZuQlXHvp2SxatKjc1c6tOqnopTmS+gL7AKPTZwG7AnemXa4H9kvrQ9Nn0vbdtAIvN694iEsaIWmKpCljrhlV6eq0G/fdczcTH5vA+RdeXDUvr7emvfLCNP7+5OOceuT+jP7lGbw0bSpjLjlr8fa6Dh0YtPM3efqvfwLgL+Pv56tfyyZS/8pGmzF/3jw++uC9SlQ9l9SSpSCn0jJiqdNdBpwINPwWXRN4LyIWpM8zgD5pvQ8wHSBtfz/t3yoVH2IYEaOAUQCfLSAqXJ124S+PT+S6saMZc/1NdO7cudLVsTLZf/hP2H/4TwD4x7NPMf6emzn8+JHMfms6PdfuR0Qw7cnHWavvugB069GLl6ZNYcfd9uHt6a8zf/48VuvStZJfIV9a0DYqzKllTiM1jMKbKmlwm9StBUo5xPAWYDDQXdIMYGREjCnV9fLqpBOOZ8rkJ3nvvbnsvusu/PjoYxl7zSjmzZ/HUUceBsBmW2zBGSPPqXBNrRIigusuOze7gRnQd8D6HPzjEwH47uE/5aYrz+fR+24FiUOPO91/tbVAGw4x3An4lqS9gU7A6sDlwBqSOqbWdl9gZtp/JtAPmCGpI9AF+HdrL66I9tP4dUvcljbpFQ9wsmUN3rDbCifw5FffLzpvtlmvS1HXSy3xEyJiX0l3AHdFxK2SrgamRcRvJB0NbBYRR0k6EPh2RBzQmu8A7aBP3MysIlrSKd46JwHHS6on6/Nu6IkYA6yZyo8HTm71FWgHfeJmZpVQiic2I2ICMCGtvwps28g+nwHD2uqaDnEzq0nVcvvAIW5mNalKMtwhbmY1qkpS3CFuZjXJs92bmeVYdUS4Q9zMalWVpLhD3MxqUrVMCuEQN7OaVCVd4g5xM6tNVZLhDnEzq03V8rIwh7iZ1aQqyXCHuJnVpirJcIe4mdWoKklxh7iZ1SQPMTQzy7EWzGLfrjnEzaw2OcTNzPLL3SlmZjlWLUMMPcemmdWktpxiU1I/SX+S9IKk5yUdl8q7SRov6eX0s2sql6QrJNVLmiZp69Z+D4e4mdWmtp0oeQHws4jYGNgeOFrSxmSTID8aEQOBR/liUuS9gIFpGQFc1dqv4RA3s5pUJxW9NCci3o6Ip9L6h8CLQB9gKHB92u16YL+0PhS4ITKTgDUk9W7V92jNQWZmede2DfGC80r9ga2AJ4BeEfF22jQL6JXW+wDTCw6bkcpazCFuZrWpBSkuaYSkKQXLiEZPKa0K3AX8V0R8ULgtIgKItv4aHp1iZjWpJUMMI2IUMKrJ80krkQX4zRFxdyr+l6TeEfF26i6ZncpnAv0KDu+bylrMLXEzq0lS8Uvz55KAMcCLEfGrgk3jgOFpfThwX0H5IWmUyvbA+wXdLi3ilriZ1aQ2Hia+E/BD4FlJz6SyU4ELgNslHQG8ARyQtj0I7A3UA58Ah7X2wg5xM6tJbTkpRET8meX/Xtitkf0DOLotru0QN7OaVC1PbDrEzawmVUmGO8TNrDa5JW5mlmN+i6GZWY65JW5mlmMOcTOzHHN3iplZnlVHhjvEzaw2VUmGO8TNrDa5T9zMLMeKmewhD/wWQzOzHHNL3MxqUpU0xB3iZlabPMTQzCzH3BI3M8uxKslwh7iZ1aa2nBSikhziZlaTqiTDHeJmVpuqJMMd4mZWo6okxR3iZlaTqmWIobJJl629kTQiIkZVuh7WfvjfhDXGj923XyMqXQFrd/xvwpbhEDczyzGHuJlZjjnE2y/3fdrS/G/CluEbm2ZmOeaWuJlZjjnEzcxyzCFeQZL8sJUtQdXyViYrG4dIBaTwvgBYSdL9EfGHStfJKk9SQ6MqJNVFxKKKVshywS3xMkstrSuA3sCTwEmSjpb0pcrWzCpJ0mHADODsStfF8sUhXn6rAVsCR0XEzcDFwAbAsIrWyipG0qrAUOBCYB9J60fEooKWudly+R9JmUXEB8DrwKGp6C/A08COktaqULWsgiLiI+CnEXE58AhwTip3d4o1yyFeGfcAW0rqnf4P/CzwOVkXi9WgiHgzrV4GrC9pDwBJHSpXK8sDh3hl/Bl4l9Qaj4ipwDZA5wrWydqBiJgFjAFOS58XSlqpsrWy9swhXgER8TZwH7CXpGGS+gOfAQsqWS+rvDQq5bfAO5Iul/S/wFaVrpe1Xw7xComIvwLnA3sBDwH3RsSTla2VVVq6obkK0BM4GHjZ/y6sKX53SoWlP5UjItwKNwAknQD0BU6KiM8rXR9r3xziZu2MH/SxlnCIm5nlmPvEzcxyzCFuZpZjDnEzsxxziJuZ5ZhDvAZJWijpGUnPSbojjUtu7bmuk/TdtD5a0sZN7DtY0o6tuMbrkrq3to4F51lc1xYcc5SkQ9L6oZLWXtF6mLUlh3ht+jQitoyITYF5wFGFG1s7WUVEHBkRLzSxy2CgxSFeKZI6RsTVEXFDKjoUcIhbu+IQt8fJXrg0WNLjksYBL0jqIOmXkiZLmibpR5C9D13SlZL+IekPZE8WkrZNkDQorQ+R9JSkv0t6NL1a4Cjgv9NfATtL6iHprnSNyZJ2SseuKekRSc9LGg00OtuNpI8kXZr2e1RSj1S+paRJqd73SOrayLFnpms+J2lUw4w66TtcJmkKcJyksySdkFrwg4CbU/33kXRvwfl2l3TPCv+vYdZCDvEallrce5G9RRFga+C4iNgAOAJ4PyK2IXs5139KGgDsD2wIbAwcQiMt6xSm1wDfiYgtgGER8TpwNXBp+ivgceDy9Hkb4DvA6HSKkcCfI2ITsjc+rrOcr/BlYEra77F0HMANZE87bp6+28hGjr0yIrZJf410BvYt2LZyRAyKiEsaCiLiTmAK8P2I2BJ4ENio4RcHcBgwdjn1NCsZT89WmzpLeiatP0721rwdgScj4rVUvgeweUEfchdgILALcEtELATekvTHRs6/PTCx4VwRMWc59fgmsHHBtJKrpwkSdgG+nY59QNLc5Ry/CLgtrd8E3C2pC7BGRDyWyq8H7mjk2G9IOhFYBegGPA/cn7bd1sj+S4iIkHQj8ANJ1wI7kP1SMysrh3ht+jS1JhdLQfpxYRFwbEQ8vNR+e7dhPeqA7SPis0bq0hpFPX4sqRPwG2BQREyXdBbQqWCXjxs9cFnXkgX/Z8Adfv+NVYK7U2x5HgZ+3PAua0kbSPoyMBH4Xuoz7w18o5FjJwG7pO4XJHVL5R+STU/X4BHg2IYPkhp+sUwke4MfkvYClunTTuqAhr8UDibrgnkfmCtp51T+Q7KulkINgf1uavkXO2JlifpHxFvAW8DpZIFuVnZuidvyjAb6A0+lm37vAPuR9VHvCrwAvAn8bekDI+IdSSPIujfqgNnA7mSt1jslDSUL758Cv5Y0jezf4kSym59nA7dIeh74a7pOYz4GtpV0errG91L5cODqNHTyVbL+6sL6vSfpGuA5YBYwucj/Tq5L5/0U2CEiPgVuBnpExItFnsOsTfkFWJZbkj6KiFUrXIcrgacjYkwl62G1yyFuuVXpEJc0leyvgd393m+rFIe4mVmO+cammVmOOcTNzHLMIW5mlmMOcTOzHHOIm5nl2P8DcPLbRekWUX4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "svf4OEXxXCzy",
        "outputId": "bbd6c7a0-2bfc-4e67-a40e-f2c9b3a3602d"
      },
      "source": [
        "print('F1 score: ', f1_score(y_test, y_pred, average='binary'))\n",
        "print('Precision: ', precision_score(y_test, y_pred, average='binary'))\n",
        "print('Recall: ', recall_score(y_test, y_pred, average='binary'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 score:  0.9798657718120806\n",
            "Precision:  0.9864864864864865\n",
            "Recall:  0.9733333333333334\n"
          ]
        }
      ]
    }
  ]
}