notebooks/2.0_train_9g.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dGH39KDm7l4Q",
        "outputId": "34f37ccf-e60c-4a8d-8d93-d605a04fdea2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2023 NVIDIA Corporation\n",
            "Built on Tue_Aug_15_22:02:13_PDT_2023\n",
            "Cuda compilation tools, release 12.2, V12.2.140\n",
            "Build cuda_12.2.r12.2/compiler.33191640_0\n"
          ]
        }
      ],
      "source": [
        "!nvcc --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9T9i1ten7l4T",
        "outputId": "e80945ac-749c-44ee-a721-676272fc0780"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Python 3.10.12\n"
          ]
        }
      ],
      "source": [
        "!python --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tLZHMlB-7l4T",
        "outputId": "c44cce22-64d4-4126-a253-05290dda4715"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fri Dec 22 01:31:12 2023       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  NVIDIA A10                     On  | 00000000:07:00.0 Off |                    0 |\n",
            "|  0%   30C    P8              15W / 150W |      4MiB / 23028MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CSBNo73x7l4U",
        "outputId": "d6649364-1e19-4b7b-89c9-43e601459172"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "               total        used        free      shared  buff/cache   available\n",
            "Mem:           196Gi       1.0Gi       194Gi       5.0Mi       1.2Gi       194Gi\n",
            "Swap:             0B          0B          0B\n"
          ]
        }
      ],
      "source": [
        "!free -h"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g_rsPkCQ7l4U",
        "outputId": "ac50d37f-0314-4442-97ec-ab871677a1f6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Is available:  True\n",
            "Current Device:  0\n",
            "Pytorch CUDA Compiled version:  12010\n",
            "Pytorch version:  <module 'torch.version' from '/home/ubuntu/.local/lib/python3.10/site-packages/torch/version.py'>\n",
            "pytorch file:  /home/ubuntu/.local/lib/python3.10/site-packages/torch/__init__.py\n"
          ]
        }
      ],
      "source": [
        " !python -c 'import torch ; print(\"Is available: \", torch.cuda.is_available()) ; print(\"Current Device: \", torch.cuda.current_device()) ; print(\"Pytorch CUDA Compiled version: \", torch._C._cuda_getCompiledVersion()) ; print(\"Pytorch version: \", torch.version) ; print(\"pytorch file: \", torch.__file__)'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "veUNmlai1ex5",
        "outputId": "f937cfa6-2107-4e5b-b923-157434583684"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting fastprogress\n",
            "  Downloading fastprogress-1.0.3-py3-none-any.whl (12 kB)\n",
            "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: fastprogress\n",
            "Successfully installed fastprogress-1.0.3\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install fastprogress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XFdOcmpO7l4X",
        "outputId": "607d0444-8eb0-4ca2-c4b1-001905c9f1ad"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting torchtext\n",
            "  Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl.metadata (7.5 kB)\n",
            "Requirement already satisfied: tqdm in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (4.66.1)\n",
            "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (2.31.0)\n",
            "Collecting torch==2.1.2 (from torchtext)\n",
            "  Downloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl.metadata (25 kB)\n",
            "Requirement already satisfied: numpy in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (1.25.2)\n",
            "Collecting torchdata==0.7.1 (from torchtext)\n",
            "  Downloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
            "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.6.0)\n",
            "Requirement already satisfied: typing-extensions in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.1.2->torchtext) (4.8.0)\n",
            "Requirement already satisfied: sympy in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (1.9)\n",
            "Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (2.4)\n",
            "Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.0.3)\n",
            "Collecting fsspec (from torch==2.1.2->torchtext)\n",
            "  Downloading fsspec-2023.12.2-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m121.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m125.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m136.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m55.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m79.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m55.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.1.0 (from torch==2.1.2->torchtext)\n",
            "  Downloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n",
            "Requirement already satisfied: urllib3>=1.25 in /usr/lib/python3/dist-packages (from torchdata==0.7.1->torchtext) (1.26.5)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->torchtext) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->torchtext) (3.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->torchtext) (2020.6.20)\n",
            "Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl (2.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m146.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m106.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m61.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2023.12.2-py3-none-any.whl (168 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m169.0/169.0 kB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m92.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25h\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: triton, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, fsspec, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, torchdata, torchtext\n",
            "Successfully installed fsspec-2023.12.2 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 torch-2.1.2 torchdata-0.7.1 torchtext-0.16.2 triton-2.1.0\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install torchtext"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ynmdzx2cjiL2"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "import torch\n",
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import numpy as np\n",
        "import h5py\n",
        "\n",
        "from fastprogress import master_bar, progress_bar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3l9xDS3Q9jR6"
      },
      "outputs": [],
      "source": [
        "# Custom char tokenizer\n",
        "def char_tokenizer(text):\n",
        "    return list(text)\n",
        "\n",
        "# Yield tokens from dataset for vocabulary building\n",
        "def yield_tokens(data):\n",
        "    for src, trg in data:\n",
        "        yield char_tokenizer(src)\n",
        "        yield char_tokenizer(trg)\n",
        "\n",
        "# Load data from CSV\n",
        "def load_data_from_csv(filename):\n",
        "    with open(filename, 'r', encoding='utf-8') as file:\n",
        "        csv_reader = csv.reader(file)\n",
        "        data = [(row[0], row[1]) for row in csv_reader]\n",
        "    return data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w03qUJ498rXk"
      },
      "outputs": [],
      "source": [
        "train_data_raw = load_data_from_csv('train.csv')\n",
        "val_data_raw = load_data_from_csv('val.csv')\n",
        "test_data_raw = load_data_from_csv('test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UDsPBMksABM4",
        "outputId": "55411ecc-7427-4ea0-ebb9-baa7fd0f60eb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Vocab length: 4131\n"
          ]
        }
      ],
      "source": [
        "vocab = build_vocab_from_iterator(yield_tokens(train_data_raw), specials=['<sos>', '<eos>', '<unk>'])\n",
        "vocab.set_default_index(vocab[\"<unk>\"])\n",
        "\n",
        "print(f\"Vocab length: {len(vocab)}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b5xcN9G1FOpX"
      },
      "outputs": [],
      "source": [
        "# save vocab\n",
        "import pickle\n",
        "\n",
        "with open('vocab.pkl', 'wb') as f:\n",
        "    pickle.dump(vocab, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S8Q2xW5T-VSv"
      },
      "outputs": [],
      "source": [
        "# Dataset class\n",
        "class CSVDataset(Dataset):\n",
        "    def __init__(self, data, vocab):\n",
        "        self.data = data\n",
        "        self.vocab = vocab\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        src, trg = self.data[index]\n",
        "        src = [self.vocab[token] for token in char_tokenizer(src)]\n",
        "        trg = [self.vocab['<sos>']] + [self.vocab[token] for token in char_tokenizer(trg)]  # prepend <sos>\n",
        "        return torch.tensor(src), torch.tensor(trg)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6poIVrbq-ivn"
      },
      "outputs": [],
      "source": [
        "# Padding function\n",
        "def collate_fn(batch):\n",
        "    src_batch, trg_batch = zip(*batch)\n",
        "    src_batch = pad_sequence(src_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    trg_batch = pad_sequence(trg_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    return src_batch, trg_batch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fT4PhLah-lkG"
      },
      "outputs": [],
      "source": [
        "# DataLoader\n",
        "BATCH_SIZE = 512\n",
        "\n",
        "train_dataset = CSVDataset(train_data_raw, vocab)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=True)\n",
        "\n",
        "val_dataset = CSVDataset(val_data_raw, vocab)\n",
        "val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)  # Usually, no need to shuffle val/test data\n",
        "\n",
        "test_dataset = CSVDataset(test_data_raw, vocab)\n",
        "test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_8NuOCx1tJdb",
        "outputId": "d5d34ac5-b797-4c01-f5fe-efd2c816f960"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([ 3,  7, 14, 16,  7,  4, 15, 21, 21, 12,  7, 16, 23,  6, 10, 11,  5, 14]) tensor([ 0, 14,  5, 41, 16,  6,  5,  7,  4, 15])\n"
          ]
        }
      ],
      "source": [
        "# check sample from dataset\n",
        "for src, trg in train_dataset:\n",
        "    print(src, trg)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YdSwSIEX-oBI",
        "outputId": "4b5b7fd9-cc29-42f7-99da-ef5fa6b673be"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([512, 42]) torch.Size([512, 31])\n"
          ]
        }
      ],
      "source": [
        "# Sample usage:\n",
        "for src, trg in train_loader:\n",
        "    # src and trg here are tokenized, batched, and padded sequences\n",
        "    print(src.shape, trg.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iY3GquRn-8dg"
      },
      "outputs": [],
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, src):\n",
        "        embedded = self.dropout(self.embedding(src))\n",
        "        outputs, (hidden, cell) = self.rnn(embedded)\n",
        "        return hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yooI-yqD_Lfr"
      },
      "outputs": [],
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, output_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.output_dim = output_dim\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, input, hidden, cell):\n",
        "        input = input.unsqueeze(0)\n",
        "        embedded = self.dropout(self.embedding(input))\n",
        "        output, (hidden, cell) = self.rnn(embedded, (hidden, cell))\n",
        "        prediction = self.fc_out(output.squeeze(0))\n",
        "        return prediction, hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EdMXVROa_P8o"
      },
      "outputs": [],
      "source": [
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, encoder, decoder, device):\n",
        "        super().__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.device = device\n",
        "\n",
        "    def forward(self, src, trg, teacher_forcing_ratio=0.5):\n",
        "        trg_len, batch_size = trg.shape\n",
        "        trg_vocab_size = self.decoder.output_dim\n",
        "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
        "        hidden, cell = self.encoder(src)\n",
        "        input = trg[0, :]\n",
        "\n",
        "        for t in range(1, trg_len):\n",
        "            output, hidden, cell = self.decoder(input, hidden, cell)\n",
        "            outputs[t] = output\n",
        "            teacher_force = torch.rand(1) < teacher_forcing_ratio\n",
        "            top1 = output.argmax(1)\n",
        "            input = trg[t] if teacher_force else top1\n",
        "\n",
        "        return outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XJgrkpYocKKV"
      },
      "outputs": [],
      "source": [
        "class EarlyStopper:\n",
        "    def __init__(self, patience=1, min_delta=0):\n",
        "        self.patience = patience\n",
        "        self.min_delta = min_delta\n",
        "        self.counter = 0\n",
        "        self.min_validation_loss = np.inf\n",
        "\n",
        "    def early_stop(self, validation_loss):\n",
        "        if validation_loss < self.min_validation_loss:\n",
        "            self.min_validation_loss = validation_loss\n",
        "            self.counter = 0\n",
        "        elif validation_loss > (self.min_validation_loss + self.min_delta):\n",
        "            self.counter += 1\n",
        "            if self.counter >= self.patience:\n",
        "                return True\n",
        "        return False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yYpAU1Fi7l4g",
        "outputId": "1c58bb59-937e-4fed-f852-11a9ea3c77f0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        },
        "id": "fUv1IEvE_V4G",
        "outputId": "c5f8bc6b-57e6-4af9-f528-85386a889daf"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
              "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      <progress value='0' class='' max='5' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/5 00:00&lt;?]\n",
              "    </div>\n",
              "    \n",
              "\n",
              "\n",
              "    <div>\n",
              "      <progress value='8755' class='' max='479175' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      1.83% [8755/479175 41:53&lt;37:31:18]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from IPython.utils import io\n",
        "\n",
        "# Assuming src and trg are your tokenized, padded source and target sequences\n",
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)\n",
        "\n",
        "lr = 0.001\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
        "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=60, eta_min=1e-5)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "early_stopper = EarlyStopper(patience=10)\n",
        "\n",
        "epochs=5\n",
        "\n",
        "mb = master_bar(range(epochs))\n",
        "mb.names = ['Training loss', 'Validation loss']\n",
        "\n",
        "x = []\n",
        "training_losses = []\n",
        "validation_losses = []\n",
        "\n",
        "valid_mean_min = np.Inf\n",
        "\n",
        "# Training loop\n",
        "for epoch in mb:\n",
        "  x.append(epoch)\n",
        "  with io.capture_output() as captured:\n",
        "      %store x\n",
        "  total_loss = torch.Tensor([0.0]).to(device)\n",
        "\n",
        "  # train\n",
        "  total_train_batches = len(train_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_train_batches\n",
        "  tbc = 0\n",
        "  model.train()\n",
        "  for batch in progress_bar(train_loader, parent=mb):\n",
        "      tbc = tbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store tbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      optimizer.zero_grad()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      total_loss += loss.item()\n",
        "\n",
        "  # decay lr\n",
        "  scheduler.step()\n",
        "  mean = total_loss / len(train_loader)\n",
        "  training_losses.append(mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store training_losses\n",
        "\n",
        "  # validation\n",
        "  model.eval()\n",
        "  total_val_batches = len(val_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_val_batches\n",
        "  validation_loss = torch.Tensor([0.0]).to(device)\n",
        "  vbc = 0\n",
        "  with torch.no_grad():\n",
        "    for batch in progress_bar(val_loader, parent=mb):\n",
        "      vbc = vbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store vbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      validation_loss += loss.item()\n",
        "\n",
        "  val_mean = validation_loss / len(val_loader)\n",
        "  validation_losses.append(val_mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store validation_losses\n",
        "  mb.update_graph([[x, training_losses], [x, validation_losses]], [0,epochs])\n",
        "  mb.write(f\"\\nEpoch {epoch}: Training loss {mean.item():.6f} validation loss {val_mean.item():.6f} with lr {lr:.6f}\")\n",
        "\n",
        "  # save model if validation loss has decreased\n",
        "  if val_mean.item() <= valid_mean_min:\n",
        "      print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
        "      valid_mean_min,\n",
        "      val_mean.item()))\n",
        "      torch.save(model.state_dict(), 'pass_predict.pt')\n",
        "      valid_mean_min = val_mean.item()\n",
        "\n",
        "  if early_stopper.early_stop(val_mean.item()):\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xYj3tlYl7l4i",
        "outputId": "974deab9-f027-40d2-91e5-2c065aab118a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "done\n"
          ]
        }
      ],
      "source": [
        "print(\"done\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U3-H2b0ckB7y"
      },
      "outputs": [],
      "source": [
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "46InKE2mJQrM",
        "outputId": "5437ff28-1eab-4140-ffbc-5e8bde095659"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.load_state_dict(torch.load('pass_predict.pt'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JxAYbzD9LXL6"
      },
      "outputs": [],
      "source": [
        "itos = vocab.get_itos()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "8JNZJ5LNET7_",
        "outputId": "c1a7bd5e-d3d7-4917-9a47-b7b1607c6ec5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'<sos>'"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "itos[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "emaRJPNtKVzN",
        "outputId": "f7a2091d-00ab-4ad2-b583-4dead569e561"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Average BLEU score: 0.719\n"
          ]
        }
      ],
      "source": [
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "\n",
        "model.eval()\n",
        "total_bleu = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off tea\n",
        "        cher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "\n",
        "        reference = [itos[tok.item()] for tok in trg[1:].reshape(-1)]\n",
        "        total_bleu += sentence_bleu([reference], predicted_sentence)\n",
        "\n",
        "average_bleu = total_bleu / len(test_loader)\n",
        "print(f'Average BLEU score: {average_bleu:.3f}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TnS01zDHKbPU"
      },
      "outputs": [],
      "source": [
        "def predict(model, vocab, device, src_sentence, n=1):\n",
        "    model.eval()\n",
        "\n",
        "    # Convert the source sentence to tokens\n",
        "    src_tokens = [tok for tok in src_sentence]\n",
        "    src_indexes = [vocab[tok] for tok in src_tokens] + [vocab['<eos>']]\n",
        "\n",
        "    # Convert to tensor and add batch dimension\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "\n",
        "    # Initialize target sequence with <sos> token\n",
        "    result = []\n",
        "    for i in range(n):\n",
        "        trg_indexes = [vocab['<sos>']]\n",
        "        predicted_sentence = []\n",
        "        # Predict next token\n",
        "        while True:\n",
        "            trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "            with torch.no_grad():\n",
        "                output = model(src_tensor, trg_tensor)\n",
        "                #print(output)\n",
        "                predicted_token = output.argmax(2)[-1, :].item()\n",
        "                # Check for end of sentence\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                if predicted_token == vocab['<eos>']:\n",
        "                    break\n",
        "\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                # Convert the target sequence to words\n",
        "                predicted_sentence.append(itos[predicted_token])\n",
        "        if ''.join(predicted_sentence[1:]) in result:\n",
        "            continue\n",
        "        result.append(''.join(predicted_sentence[1:]))\n",
        "\n",
        "    return result  # Exclude the <sos> token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IkZo16IbZjdZ",
        "outputId": "dc1a5231-a8cf-4d1e-ee00-634291fe0761"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['ean',\n",
              " 'ara',\n",
              " 'eera',\n",
              " 'erae',\n",
              " 'aer',\n",
              " 'ann',\n",
              " 'era',\n",
              " 'err',\n",
              " 'aan',\n",
              " 'aee',\n",
              " 'anr',\n",
              " 'aar',\n",
              " 'een',\n",
              " 'ear',\n",
              " 'ana',\n",
              " 'ere',\n",
              " 'eer',\n",
              " 'er',\n",
              " 'ari',\n",
              " 'erern',\n",
              " 'eae',\n",
              " 'erne',\n",
              " 'aen',\n",
              " 'ane',\n",
              " 'ern',\n",
              " 'ea',\n",
              " 'eria',\n",
              " 'ar',\n",
              " 'eeaa',\n",
              " 'aara',\n",
              " 'aaa',\n",
              " 'erin',\n",
              " 'are',\n",
              " 'eaa',\n",
              " 'arae',\n",
              " 'aa1',\n",
              " 'aae',\n",
              " 'aena',\n",
              " 'erre',\n",
              " 'erer',\n",
              " 'eri']"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predict(model, vocab, device, \"pyro.archvile@googlemail.com\", 100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zp6-T-dBMk4-",
        "outputId": "a319843d-c432-4f2f-cc58-71395d793554"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "arturo100@rubi.net,tequiero1\n",
            "mahei001@mail.ru,йцувыфячс\n",
            "machomiscellanykml4@yandex.ru,iEW843lGo\n",
            "angieg@tm.net.my,c_sunstrom\n",
            "js152008@yahoo.com,boots5\n",
            "jenia.kuznetzova@hotmail.ru,jenia12345jenia\n",
            "pucoy_gunawan@yahoo.com,Janganbuka123\n",
            "naidu35pharma@gmail.com,malleswari\n",
            "tailincoln@cox.net,misrules26\n",
            "jayeshjagetiya83@gmail.com,969858\n"
          ]
        }
      ],
      "source": [
        "!head -10 test.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SkAImySYMZe8",
        "outputId": "2506c113-c2f2-4974-ba42-3e44329094f2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[26, 7, 3, 5, 15, 19, 27, 12, 20, 3, 18, 4, 4, 10, 13, 4, 8]\n",
            "tensor([[[0., 0., 0.,  ..., 0., 0., 0.]]], device='cuda:0')\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "src_sentence = \"grae123@yahoo.com\"\n",
        "src_tokens = [tok for tok in src_sentence]\n",
        "src_indexes = [vocab[tok] for tok in src_tokens]\n",
        "print(src_indexes)\n",
        "src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "trg_indexes = [vocab['<sos>']]\n",
        "trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "with torch.no_grad():\n",
        "  output = model(src_tensor, trg_tensor)\n",
        "print(output)\n",
        "output_dim = output.shape[-1]\n",
        "output = output[1:].reshape(-1, output_dim)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L4oj-Xtdr8Rq",
        "outputId": "8da9c9bd-4ed4-4f85-97f4-e0ec07054b36"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predicted_tokens = torch.argmax(output, dim=1)\n",
        "\n",
        "predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "predicted_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pP2LRT9e7l4p"
      },
      "outputs": [],
      "source": [
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off teacher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = ''.join([itos[tok.item()] for tok in predicted_tokens])\n",
        "\n",
        "        reference = ''.join([itos[tok.item()] for tok in trg[1:].reshape(-1)])\n",
        "        print(predicted_sentence)\n",
        "        print(reference)\n",
        "        break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f_A2MPrT7l4p",
        "outputId": "fd80888c-8b6c-46f1-d8f2-d75605a745f9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2023 NVIDIA Corporation\n",
            "Built on Tue_Aug_15_22:02:13_PDT_2023\n",
            "Cuda compilation tools, release 12.2, V12.2.140\n",
            "Build cuda_12.2.r12.2/compiler.33191640_0\n"
          ]
        }
      ],
      "source": [
        "!nvcc --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8ZNRpfMH7l4q",
        "outputId": "ce019290-8b50-425a-9f9c-b0481c189f45"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Python 3.10.12\n"
          ]
        }
      ],
      "source": [
        "!python --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U9sLbzWg7l4q",
        "outputId": "e8fcaf77-5573-4706-8dc8-634170d80aad"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fri Dec 22 01:31:12 2023       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  NVIDIA A10                     On  | 00000000:07:00.0 Off |                    0 |\n",
            "|  0%   30C    P8              15W / 150W |      4MiB / 23028MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KSHTQbNY7l4q",
        "outputId": "f3a830f0-835f-494f-ab8d-fc95aab6ce84"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "               total        used        free      shared  buff/cache   available\n",
            "Mem:           196Gi       1.0Gi       194Gi       5.0Mi       1.2Gi       194Gi\n",
            "Swap:             0B          0B          0B\n"
          ]
        }
      ],
      "source": [
        "!free -h"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OsmSJc557l4_",
        "outputId": "9a810ed3-7117-464c-da36-4d6000335d16"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Is available:  True\n",
            "Current Device:  0\n",
            "Pytorch CUDA Compiled version:  12010\n",
            "Pytorch version:  <module 'torch.version' from '/home/ubuntu/.local/lib/python3.10/site-packages/torch/version.py'>\n",
            "pytorch file:  /home/ubuntu/.local/lib/python3.10/site-packages/torch/__init__.py\n"
          ]
        }
      ],
      "source": [
        " !python -c 'import torch ; print(\"Is available: \", torch.cuda.is_available()) ; print(\"Current Device: \", torch.cuda.current_device()) ; print(\"Pytorch CUDA Compiled version: \", torch._C._cuda_getCompiledVersion()) ; print(\"Pytorch version: \", torch.version) ; print(\"pytorch file: \", torch.__file__)'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f937cfa6-2107-4e5b-b923-157434583684",
        "id": "0BQtJHlW7l5A"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting fastprogress\n",
            "  Downloading fastprogress-1.0.3-py3-none-any.whl (12 kB)\n",
            "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: fastprogress\n",
            "Successfully installed fastprogress-1.0.3\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install fastprogress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BkwFT2wB7l5A",
        "outputId": "9a636e12-9c04-45d0-a4eb-da5cce962a50"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting torchtext\n",
            "  Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl.metadata (7.5 kB)\n",
            "Requirement already satisfied: tqdm in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (4.66.1)\n",
            "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (2.31.0)\n",
            "Collecting torch==2.1.2 (from torchtext)\n",
            "  Downloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl.metadata (25 kB)\n",
            "Requirement already satisfied: numpy in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (1.25.2)\n",
            "Collecting torchdata==0.7.1 (from torchtext)\n",
            "  Downloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
            "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.6.0)\n",
            "Requirement already satisfied: typing-extensions in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.1.2->torchtext) (4.8.0)\n",
            "Requirement already satisfied: sympy in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (1.9)\n",
            "Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (2.4)\n",
            "Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.0.3)\n",
            "Collecting fsspec (from torch==2.1.2->torchtext)\n",
            "  Downloading fsspec-2023.12.2-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m121.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m125.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m136.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m55.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m79.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m55.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.1.0 (from torch==2.1.2->torchtext)\n",
            "  Downloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n",
            "Requirement already satisfied: urllib3>=1.25 in /usr/lib/python3/dist-packages (from torchdata==0.7.1->torchtext) (1.26.5)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->torchtext) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->torchtext) (3.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->torchtext) (2020.6.20)\n",
            "Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl (2.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m146.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m106.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m61.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2023.12.2-py3-none-any.whl (168 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m169.0/169.0 kB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m92.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25h\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: triton, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, fsspec, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, torchdata, torchtext\n",
            "Successfully installed fsspec-2023.12.2 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 torch-2.1.2 torchdata-0.7.1 torchtext-0.16.2 triton-2.1.0\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install torchtext"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UWdMF7SC7l5B"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "import torch\n",
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import numpy as np\n",
        "import h5py\n",
        "\n",
        "from fastprogress import master_bar, progress_bar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mesdpDnZ7l5B"
      },
      "outputs": [],
      "source": [
        "# Custom char tokenizer\n",
        "def char_tokenizer(text):\n",
        "    return list(text)\n",
        "\n",
        "# Yield tokens from dataset for vocabulary building\n",
        "def yield_tokens(data):\n",
        "    for src, trg in data:\n",
        "        yield char_tokenizer(src)\n",
        "        yield char_tokenizer(trg)\n",
        "\n",
        "# Load data from CSV\n",
        "def load_data_from_csv(filename):\n",
        "    with open(filename, 'r', encoding='utf-8') as file:\n",
        "        csv_reader = csv.reader(file)\n",
        "        data = [(row[0], row[1]) for row in csv_reader]\n",
        "    return data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nmSkGS_K7l5C"
      },
      "outputs": [],
      "source": [
        "train_data_raw = load_data_from_csv('train.csv')\n",
        "val_data_raw = load_data_from_csv('val.csv')\n",
        "test_data_raw = load_data_from_csv('test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k4vMBqz97l5C",
        "outputId": "3e5677b1-4ab6-4c8e-eb3d-5a9122aced8a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Vocab length: 4131\n"
          ]
        }
      ],
      "source": [
        "vocab = build_vocab_from_iterator(yield_tokens(train_data_raw), specials=['<sos>', '<eos>', '<unk>'])\n",
        "vocab.set_default_index(vocab[\"<unk>\"])\n",
        "\n",
        "print(f\"Vocab length: {len(vocab)}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AEthyGFQ7l5C"
      },
      "outputs": [],
      "source": [
        "# save vocab\n",
        "import pickle\n",
        "\n",
        "with open('vocab.pkl', 'wb') as f:\n",
        "    pickle.dump(vocab, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m-28C_VU7l5D"
      },
      "outputs": [],
      "source": [
        "# Dataset class\n",
        "class CSVDataset(Dataset):\n",
        "    def __init__(self, data, vocab):\n",
        "        self.data = data\n",
        "        self.vocab = vocab\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        src, trg = self.data[index]\n",
        "        src = [self.vocab[token] for token in char_tokenizer(src)]\n",
        "        trg = [self.vocab['<sos>']] + [self.vocab[token] for token in char_tokenizer(trg)]  # prepend <sos>\n",
        "        return torch.tensor(src), torch.tensor(trg)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9M0hSGf67l5D"
      },
      "outputs": [],
      "source": [
        "# Padding function\n",
        "def collate_fn(batch):\n",
        "    src_batch, trg_batch = zip(*batch)\n",
        "    src_batch = pad_sequence(src_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    trg_batch = pad_sequence(trg_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    return src_batch, trg_batch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tZeqBqMk7l5E"
      },
      "outputs": [],
      "source": [
        "# DataLoader\n",
        "BATCH_SIZE = 512\n",
        "\n",
        "train_dataset = CSVDataset(train_data_raw, vocab)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=True)\n",
        "\n",
        "val_dataset = CSVDataset(val_data_raw, vocab)\n",
        "val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)  # Usually, no need to shuffle val/test data\n",
        "\n",
        "test_dataset = CSVDataset(test_data_raw, vocab)\n",
        "test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d5d34ac5-b797-4c01-f5fe-efd2c816f960",
        "id": "rxKRtOjF7l5F"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([ 3,  7, 14, 16,  7,  4, 15, 21, 21, 12,  7, 16, 23,  6, 10, 11,  5, 14]) tensor([ 0, 14,  5, 41, 16,  6,  5,  7,  4, 15])\n"
          ]
        }
      ],
      "source": [
        "# check sample from dataset\n",
        "for src, trg in train_dataset:\n",
        "    print(src, trg)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4b5b7fd9-cc29-42f7-99da-ef5fa6b673be",
        "id": "R9g30Mu_7l5G"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([512, 42]) torch.Size([512, 31])\n"
          ]
        }
      ],
      "source": [
        "# Sample usage:\n",
        "for src, trg in train_loader:\n",
        "    # src and trg here are tokenized, batched, and padded sequences\n",
        "    print(src.shape, trg.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cjqwE6Eu7l5G"
      },
      "outputs": [],
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, src):\n",
        "        embedded = self.dropout(self.embedding(src))\n",
        "        outputs, (hidden, cell) = self.rnn(embedded)\n",
        "        return hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "okSqJYJS7l5H"
      },
      "outputs": [],
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, output_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.output_dim = output_dim\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, input, hidden, cell):\n",
        "        input = input.unsqueeze(0)\n",
        "        embedded = self.dropout(self.embedding(input))\n",
        "        output, (hidden, cell) = self.rnn(embedded, (hidden, cell))\n",
        "        prediction = self.fc_out(output.squeeze(0))\n",
        "        return prediction, hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SivEEaFw7l5J"
      },
      "outputs": [],
      "source": [
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, encoder, decoder, device):\n",
        "        super().__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.device = device\n",
        "\n",
        "    def forward(self, src, trg, teacher_forcing_ratio=0.5):\n",
        "        trg_len, batch_size = trg.shape\n",
        "        trg_vocab_size = self.decoder.output_dim\n",
        "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
        "        hidden, cell = self.encoder(src)\n",
        "        input = trg[0, :]\n",
        "\n",
        "        for t in range(1, trg_len):\n",
        "            output, hidden, cell = self.decoder(input, hidden, cell)\n",
        "            outputs[t] = output\n",
        "            teacher_force = torch.rand(1) < teacher_forcing_ratio\n",
        "            top1 = output.argmax(1)\n",
        "            input = trg[t] if teacher_force else top1\n",
        "\n",
        "        return outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rOdjOCUd7l5K"
      },
      "outputs": [],
      "source": [
        "class EarlyStopper:\n",
        "    def __init__(self, patience=1, min_delta=0):\n",
        "        self.patience = patience\n",
        "        self.min_delta = min_delta\n",
        "        self.counter = 0\n",
        "        self.min_validation_loss = np.inf\n",
        "\n",
        "    def early_stop(self, validation_loss):\n",
        "        if validation_loss < self.min_validation_loss:\n",
        "            self.min_validation_loss = validation_loss\n",
        "            self.counter = 0\n",
        "        elif validation_loss > (self.min_validation_loss + self.min_delta):\n",
        "            self.counter += 1\n",
        "            if self.counter >= self.patience:\n",
        "                return True\n",
        "        return False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JO5xQdC-7l5M",
        "outputId": "b65303b2-ad82-458e-d297-fbd43530ffd4"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        },
        "outputId": "c5f8bc6b-57e6-4af9-f528-85386a889daf",
        "id": "RZgLV1yJ7l5N"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
              "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      <progress value='0' class='' max='5' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/5 00:00&lt;?]\n",
              "    </div>\n",
              "    \n",
              "\n",
              "\n",
              "    <div>\n",
              "      <progress value='39495' class='' max='479175' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      8.24% [39495/479175 2:46:54&lt;30:58:07]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from IPython.utils import io\n",
        "\n",
        "# Assuming src and trg are your tokenized, padded source and target sequences\n",
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)\n",
        "\n",
        "lr = 0.001\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
        "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=60, eta_min=1e-5)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "early_stopper = EarlyStopper(patience=10)\n",
        "\n",
        "epochs=5\n",
        "\n",
        "mb = master_bar(range(epochs))\n",
        "mb.names = ['Training loss', 'Validation loss']\n",
        "\n",
        "x = []\n",
        "training_losses = []\n",
        "validation_losses = []\n",
        "\n",
        "valid_mean_min = np.Inf\n",
        "\n",
        "# Training loop\n",
        "for epoch in mb:\n",
        "  x.append(epoch)\n",
        "  with io.capture_output() as captured:\n",
        "      %store x\n",
        "  total_loss = torch.Tensor([0.0]).to(device)\n",
        "\n",
        "  # train\n",
        "  total_train_batches = len(train_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_train_batches\n",
        "  tbc = 0\n",
        "  model.train()\n",
        "  for batch in progress_bar(train_loader, parent=mb):\n",
        "      tbc = tbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store tbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      optimizer.zero_grad()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      total_loss += loss.item()\n",
        "\n",
        "  # decay lr\n",
        "  scheduler.step()\n",
        "  mean = total_loss / len(train_loader)\n",
        "  training_losses.append(mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store training_losses\n",
        "\n",
        "  # validation\n",
        "  model.eval()\n",
        "  total_val_batches = len(val_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_val_batches\n",
        "  validation_loss = torch.Tensor([0.0]).to(device)\n",
        "  vbc = 0\n",
        "  with torch.no_grad():\n",
        "    for batch in progress_bar(val_loader, parent=mb):\n",
        "      vbc = vbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store vbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      validation_loss += loss.item()\n",
        "\n",
        "  val_mean = validation_loss / len(val_loader)\n",
        "  validation_losses.append(val_mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store validation_losses\n",
        "  mb.update_graph([[x, training_losses], [x, validation_losses]], [0,epochs])\n",
        "  mb.write(f\"\\nEpoch {epoch}: Training loss {mean.item():.6f} validation loss {val_mean.item():.6f} with lr {lr:.6f}\")\n",
        "\n",
        "  # save model if validation loss has decreased\n",
        "  if val_mean.item() <= valid_mean_min:\n",
        "      print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
        "      valid_mean_min,\n",
        "      val_mean.item()))\n",
        "      torch.save(model.state_dict(), '/content/drive/MyDrive/Colab/password_predict/pass_predict.pt')\n",
        "      valid_mean_min = val_mean.item()\n",
        "\n",
        "  if early_stopper.early_stop(val_mean.item()):\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "geDd2Wa67l5O"
      },
      "outputs": [],
      "source": [
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5437ff28-1eab-4140-ffbc-5e8bde095659",
        "id": "o2H3MCtW7l5O"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.load_state_dict(torch.load('/content/drive/MyDrive/Colab/password_predict/pass_predict.pt'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XeYJr89w7l5P"
      },
      "outputs": [],
      "source": [
        "itos = vocab.get_itos()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c1a7bd5e-d3d7-4917-9a47-b7b1607c6ec5",
        "id": "qJcjIOk47l5P"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<sos>'"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "itos[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7a2091d-00ab-4ad2-b583-4dead569e561",
        "id": "q-p-0llU7l5Q"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Average BLEU score: 0.719\n"
          ]
        }
      ],
      "source": [
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "\n",
        "model.eval()\n",
        "total_bleu = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off teacher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "\n",
        "        reference = [itos[tok.item()] for tok in trg[1:].reshape(-1)]\n",
        "        total_bleu += sentence_bleu([reference], predicted_sentence)\n",
        "\n",
        "average_bleu = total_bleu / len(test_loader)\n",
        "print(f'Average BLEU score: {average_bleu:.3f}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "etdoCs-I7l5Q"
      },
      "outputs": [],
      "source": [
        "def predict(model, vocab, device, src_sentence, n=1):\n",
        "    model.eval()\n",
        "\n",
        "    # Convert the source sentence to tokens\n",
        "    src_tokens = [tok for tok in src_sentence]\n",
        "    src_indexes = [vocab[tok] for tok in src_tokens] + [vocab['<eos>']]\n",
        "\n",
        "    # Convert to tensor and add batch dimension\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "\n",
        "    # Initialize target sequence with <sos> token\n",
        "    result = []\n",
        "    for i in range(n):\n",
        "        trg_indexes = [vocab['<sos>']]\n",
        "        predicted_sentence = []\n",
        "        # Predict next token\n",
        "        while True:\n",
        "            trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "            with torch.no_grad():\n",
        "                output = model(src_tensor, trg_tensor)\n",
        "                #print(output)\n",
        "                predicted_token = output.argmax(2)[-1, :].item()\n",
        "                # Check for end of sentence\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                if predicted_token == vocab['<eos>']:\n",
        "                    break\n",
        "\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                # Convert the target sequence to words\n",
        "                predicted_sentence.append(itos[predicted_token])\n",
        "        if ''.join(predicted_sentence[1:]) in result:\n",
        "            continue\n",
        "        result.append(''.join(predicted_sentence[1:]))\n",
        "\n",
        "    return result  # Exclude the <sos> token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc1a5231-a8cf-4d1e-ee00-634291fe0761",
        "id": "hPZQkT0o7l5R"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['aere',\n",
              " 'aels',\n",
              " 'ailer',\n",
              " 'aell',\n",
              " 'ana',\n",
              " 'aie',\n",
              " 'aea',\n",
              " 'ail',\n",
              " 'anc',\n",
              " 'aia',\n",
              " 'aee',\n",
              " 'ain',\n",
              " 'aeon',\n",
              " 'aio',\n",
              " 'ane',\n",
              " 'aear',\n",
              " 'ano',\n",
              " 'aier',\n",
              " 'aeer',\n",
              " 'anar',\n",
              " 'ance',\n",
              " 'aic',\n",
              " 'aer',\n",
              " 'aera',\n",
              " 'ael',\n",
              " 'anan']"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predict(model, vocab, device, \"pyro.archvile@googlemail.com\", 100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a319843d-c432-4f2f-cc58-71395d793554",
        "id": "VgqkMGii7l5S"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "user,pass\n",
            "bikramojha@yandex.ru,bhp455614\n",
            "adjoa.mansa@yahoo.com,rbsfdplc\n",
            "pyro.archvile@googlemail.com,Omega666\n",
            "red_head76_2000@yahoo.com,0002_67daeh_der\n",
            "laurentnancy@hotmail.fr,27091978zxc\n",
            "19ol64@mail.ru,785333\n",
            "grae123@yahoo.com,enzofan1\n",
            "gulnora_nabiyeva@yahoo.com,1qaz2wsx3edc\n",
            "wydik@ya.ru,525252\n"
          ]
        }
      ],
      "source": [
        "!head -10 /content/drive/MyDrive/Colab/password_predict/data/test.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2506c113-c2f2-4974-ba42-3e44329094f2",
        "id": "uHd-xHTV7l5T"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[26, 7, 3, 6, 15, 20, 27, 12, 19, 3, 18, 4, 4, 10, 13, 4, 8]\n",
            "tensor([[[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]], device='cuda:0')\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "src_sentence = \"grae123@yahoo.com\"\n",
        "src_tokens = [tok for tok in src_sentence]\n",
        "src_indexes = [vocab[tok] for tok in src_tokens]\n",
        "print(src_indexes)\n",
        "src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "trg_indexes = [vocab['<sos>']]\n",
        "trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "with torch.no_grad():\n",
        "  output = model(src_tensor, trg_tensor)\n",
        "print(output)\n",
        "output_dim = output.shape[-1]\n",
        "output = output[1:].reshape(-1, output_dim)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8da9c9bd-4ed4-4f85-97f4-e0ec07054b36",
        "id": "_EeclF4W7l5T"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predicted_tokens = torch.argmax(output, dim=1)\n",
        "\n",
        "predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "predicted_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rTBFuTK8uGe6"
      },
      "outputs": [],
      "source": [
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off teacher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = ''.join([itos[tok.item()] for tok in predicted_tokens])\n",
        "\n",
        "        reference = ''.join([itos[tok.item()] for tok in trg[1:].reshape(-1)])\n",
        "        print(predicted_sentence)\n",
        "        print(reference)\n",
        "        break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X7kES4HTP3m-"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eWmQbqpS7l5V",
        "outputId": "fd9718ef-711e-4c04-c0e0-540f3a6cadbd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2023 NVIDIA Corporation\n",
            "Built on Tue_Aug_15_22:02:13_PDT_2023\n",
            "Cuda compilation tools, release 12.2, V12.2.140\n",
            "Build cuda_12.2.r12.2/compiler.33191640_0\n"
          ]
        }
      ],
      "source": [
        "!nvcc --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uK7LBvow7l5V",
        "outputId": "fc73e79b-66c4-42e7-d76f-4d9d4dd72ccb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Python 3.10.12\n"
          ]
        }
      ],
      "source": [
        "!python --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SbPwOZlJ7l5W",
        "outputId": "e4221bf9-b9b2-4295-a0ef-7e842fc074cb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fri Dec 22 01:31:12 2023       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  NVIDIA A10                     On  | 00000000:07:00.0 Off |                    0 |\n",
            "|  0%   30C    P8              15W / 150W |      4MiB / 23028MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eTRQE7J97l5W",
        "outputId": "9f75b1f8-78e0-4a9a-c15e-4935f2308aaf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "               total        used        free      shared  buff/cache   available\n",
            "Mem:           196Gi       1.0Gi       194Gi       5.0Mi       1.2Gi       194Gi\n",
            "Swap:             0B          0B          0B\n"
          ]
        }
      ],
      "source": [
        "!free -h"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E87CQs5x7l5W",
        "outputId": "f4a77db8-5f5b-45a0-bcd7-db99aa2d5dd4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Is available:  True\n",
            "Current Device:  0\n",
            "Pytorch CUDA Compiled version:  12010\n",
            "Pytorch version:  <module 'torch.version' from '/home/ubuntu/.local/lib/python3.10/site-packages/torch/version.py'>\n",
            "pytorch file:  /home/ubuntu/.local/lib/python3.10/site-packages/torch/__init__.py\n"
          ]
        }
      ],
      "source": [
        " !python -c 'import torch ; print(\"Is available: \", torch.cuda.is_available()) ; print(\"Current Device: \", torch.cuda.current_device()) ; print(\"Pytorch CUDA Compiled version: \", torch._C._cuda_getCompiledVersion()) ; print(\"Pytorch version: \", torch.version) ; print(\"pytorch file: \", torch.__file__)'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f937cfa6-2107-4e5b-b923-157434583684",
        "id": "oIlYmrtG7l5X"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting fastprogress\n",
            "  Downloading fastprogress-1.0.3-py3-none-any.whl (12 kB)\n",
            "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: fastprogress\n",
            "Successfully installed fastprogress-1.0.3\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install fastprogress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Op5Lr1iL7l5Y",
        "outputId": "0af1a768-d580-40d5-ece8-edc09bb2cae9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting torchtext\n",
            "  Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl.metadata (7.5 kB)\n",
            "Requirement already satisfied: tqdm in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (4.66.1)\n",
            "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (2.31.0)\n",
            "Collecting torch==2.1.2 (from torchtext)\n",
            "  Downloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl.metadata (25 kB)\n",
            "Requirement already satisfied: numpy in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (1.25.2)\n",
            "Collecting torchdata==0.7.1 (from torchtext)\n",
            "  Downloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
            "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.6.0)\n",
            "Requirement already satisfied: typing-extensions in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.1.2->torchtext) (4.8.0)\n",
            "Requirement already satisfied: sympy in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (1.9)\n",
            "Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (2.4)\n",
            "Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.0.3)\n",
            "Collecting fsspec (from torch==2.1.2->torchtext)\n",
            "  Downloading fsspec-2023.12.2-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m121.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m125.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m136.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m55.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m79.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m55.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.1.0 (from torch==2.1.2->torchtext)\n",
            "  Downloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n",
            "Requirement already satisfied: urllib3>=1.25 in /usr/lib/python3/dist-packages (from torchdata==0.7.1->torchtext) (1.26.5)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->torchtext) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->torchtext) (3.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->torchtext) (2020.6.20)\n",
            "Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl (2.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m146.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m106.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m61.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2023.12.2-py3-none-any.whl (168 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m169.0/169.0 kB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m92.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25h\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: triton, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, fsspec, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, torchdata, torchtext\n",
            "Successfully installed fsspec-2023.12.2 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 torch-2.1.2 torchdata-0.7.1 torchtext-0.16.2 triton-2.1.0\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install torchtext"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-xAVCW-v7l5Y"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "import torch\n",
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import numpy as np\n",
        "import h5py\n",
        "\n",
        "from fastprogress import master_bar, progress_bar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FfXl7Gyc7l5Z"
      },
      "outputs": [],
      "source": [
        "# Custom char tokenizer\n",
        "def char_tokenizer(text):\n",
        "    return list(text)\n",
        "\n",
        "# Yield tokens from dataset for vocabulary building\n",
        "def yield_tokens(data):\n",
        "    for src, trg in data:\n",
        "        yield char_tokenizer(src)\n",
        "        yield char_tokenizer(trg)\n",
        "\n",
        "# Load data from CSV\n",
        "def load_data_from_csv(filename):\n",
        "    with open(filename, 'r', encoding='utf-8') as file:\n",
        "        csv_reader = csv.reader(file)\n",
        "        data = [(row[0], row[1]) for row in csv_reader]\n",
        "    return data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i2KXLZca7l5Z"
      },
      "outputs": [],
      "source": [
        "train_data_raw = load_data_from_csv('train.csv')\n",
        "val_data_raw = load_data_from_csv('val.csv')\n",
        "test_data_raw = load_data_from_csv('test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h_cGAFSH7l5a",
        "outputId": "e6c610e0-ae4b-45c4-e124-bd4339c7249a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Vocab length: 4131\n"
          ]
        }
      ],
      "source": [
        "vocab = build_vocab_from_iterator(yield_tokens(train_data_raw), specials=['<sos>', '<eos>', '<unk>'])\n",
        "vocab.set_default_index(vocab[\"<unk>\"])\n",
        "\n",
        "print(f\"Vocab length: {len(vocab)}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PzLVDplR7l5a"
      },
      "outputs": [],
      "source": [
        "# save vocab\n",
        "import pickle\n",
        "\n",
        "with open('vocab.pkl', 'wb') as f:\n",
        "    pickle.dump(vocab, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p4J3wLG57l5b"
      },
      "outputs": [],
      "source": [
        "# Dataset class\n",
        "class CSVDataset(Dataset):\n",
        "    def __init__(self, data, vocab):\n",
        "        self.data = data\n",
        "        self.vocab = vocab\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        src, trg = self.data[index]\n",
        "        src = [self.vocab[token] for token in char_tokenizer(src)]\n",
        "        trg = [self.vocab['<sos>']] + [self.vocab[token] for token in char_tokenizer(trg)]  # prepend <sos>\n",
        "        return torch.tensor(src), torch.tensor(trg)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ekkZ2jJ47l5b"
      },
      "outputs": [],
      "source": [
        "# Padding function\n",
        "def collate_fn(batch):\n",
        "    src_batch, trg_batch = zip(*batch)\n",
        "    src_batch = pad_sequence(src_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    trg_batch = pad_sequence(trg_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    return src_batch, trg_batch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PU5Xel_w7l5b"
      },
      "outputs": [],
      "source": [
        "# DataLoader\n",
        "BATCH_SIZE = 512\n",
        "\n",
        "train_dataset = CSVDataset(train_data_raw, vocab)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=True)\n",
        "\n",
        "val_dataset = CSVDataset(val_data_raw, vocab)\n",
        "val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)  # Usually, no need to shuffle val/test data\n",
        "\n",
        "test_dataset = CSVDataset(test_data_raw, vocab)\n",
        "test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d5d34ac5-b797-4c01-f5fe-efd2c816f960",
        "id": "amRLqGwh7l5c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([ 3,  7, 14, 16,  7,  4, 15, 21, 21, 12,  7, 16, 23,  6, 10, 11,  5, 14]) tensor([ 0, 14,  5, 41, 16,  6,  5,  7,  4, 15])\n"
          ]
        }
      ],
      "source": [
        "# check sample from dataset\n",
        "for src, trg in train_dataset:\n",
        "    print(src, trg)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4b5b7fd9-cc29-42f7-99da-ef5fa6b673be",
        "id": "6uX7FGKo7l5d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([512, 42]) torch.Size([512, 31])\n"
          ]
        }
      ],
      "source": [
        "# Sample usage:\n",
        "for src, trg in train_loader:\n",
        "    # src and trg here are tokenized, batched, and padded sequences\n",
        "    print(src.shape, trg.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UWKwjKKh7l5d"
      },
      "outputs": [],
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, src):\n",
        "        embedded = self.dropout(self.embedding(src))\n",
        "        outputs, (hidden, cell) = self.rnn(embedded)\n",
        "        return hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2BH_LE6I7l5e"
      },
      "outputs": [],
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, output_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.output_dim = output_dim\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, input, hidden, cell):\n",
        "        input = input.unsqueeze(0)\n",
        "        embedded = self.dropout(self.embedding(input))\n",
        "        output, (hidden, cell) = self.rnn(embedded, (hidden, cell))\n",
        "        prediction = self.fc_out(output.squeeze(0))\n",
        "        return prediction, hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IXFlXjBb7l5e"
      },
      "outputs": [],
      "source": [
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, encoder, decoder, device):\n",
        "        super().__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.device = device\n",
        "\n",
        "    def forward(self, src, trg, teacher_forcing_ratio=0.5):\n",
        "        trg_len, batch_size = trg.shape\n",
        "        trg_vocab_size = self.decoder.output_dim\n",
        "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
        "        hidden, cell = self.encoder(src)\n",
        "        input = trg[0, :]\n",
        "\n",
        "        for t in range(1, trg_len):\n",
        "            output, hidden, cell = self.decoder(input, hidden, cell)\n",
        "            outputs[t] = output\n",
        "            teacher_force = torch.rand(1) < teacher_forcing_ratio\n",
        "            top1 = output.argmax(1)\n",
        "            input = trg[t] if teacher_force else top1\n",
        "\n",
        "        return outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AJevRtpj7l5f"
      },
      "outputs": [],
      "source": [
        "class EarlyStopper:\n",
        "    def __init__(self, patience=1, min_delta=0):\n",
        "        self.patience = patience\n",
        "        self.min_delta = min_delta\n",
        "        self.counter = 0\n",
        "        self.min_validation_loss = np.inf\n",
        "\n",
        "    def early_stop(self, validation_loss):\n",
        "        if validation_loss < self.min_validation_loss:\n",
        "            self.min_validation_loss = validation_loss\n",
        "            self.counter = 0\n",
        "        elif validation_loss > (self.min_validation_loss + self.min_delta):\n",
        "            self.counter += 1\n",
        "            if self.counter >= self.patience:\n",
        "                return True\n",
        "        return False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kXg13qYz7l5f",
        "outputId": "da27d7fd-0bd7-442c-b616-2286b1657f5b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        },
        "outputId": "c5f8bc6b-57e6-4af9-f528-85386a889daf",
        "id": "ocL3GaWr7l5g"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
              "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      <progress value='0' class='' max='5' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/5 00:00&lt;?]\n",
              "    </div>\n",
              "    \n",
              "\n",
              "\n",
              "    <div>\n",
              "      <progress value='8755' class='' max='479175' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      1.83% [8755/479175 41:53&lt;37:31:18]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from IPython.utils import io\n",
        "\n",
        "# Assuming src and trg are your tokenized, padded source and target sequences\n",
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)\n",
        "\n",
        "lr = 0.001\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
        "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=60, eta_min=1e-5)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "early_stopper = EarlyStopper(patience=10)\n",
        "\n",
        "epochs=5\n",
        "\n",
        "mb = master_bar(range(epochs))\n",
        "mb.names = ['Training loss', 'Validation loss']\n",
        "\n",
        "x = []\n",
        "training_losses = []\n",
        "validation_losses = []\n",
        "\n",
        "valid_mean_min = np.Inf\n",
        "\n",
        "# Training loop\n",
        "for epoch in mb:\n",
        "  x.append(epoch)\n",
        "  with io.capture_output() as captured:\n",
        "      %store x\n",
        "  total_loss = torch.Tensor([0.0]).to(device)\n",
        "\n",
        "  # train\n",
        "  total_train_batches = len(train_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_train_batches\n",
        "  tbc = 0\n",
        "  model.train()\n",
        "  for batch in progress_bar(train_loader, parent=mb):\n",
        "      tbc = tbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store tbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      optimizer.zero_grad()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      total_loss += loss.item()\n",
        "\n",
        "  # decay lr\n",
        "  scheduler.step()\n",
        "  mean = total_loss / len(train_loader)\n",
        "  training_losses.append(mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store training_losses\n",
        "\n",
        "  # validation\n",
        "  model.eval()\n",
        "  total_val_batches = len(val_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_val_batches\n",
        "  validation_loss = torch.Tensor([0.0]).to(device)\n",
        "  vbc = 0\n",
        "  with torch.no_grad():\n",
        "    for batch in progress_bar(val_loader, parent=mb):\n",
        "      vbc = vbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store vbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      validation_loss += loss.item()\n",
        "\n",
        "  val_mean = validation_loss / len(val_loader)\n",
        "  validation_losses.append(val_mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store validation_losses\n",
        "  mb.update_graph([[x, training_losses], [x, validation_losses]], [0,epochs])\n",
        "  mb.write(f\"\\nEpoch {epoch}: Training loss {mean.item():.6f} validation loss {val_mean.item():.6f} with lr {lr:.6f}\")\n",
        "\n",
        "  # save model if validation loss has decreased\n",
        "  if val_mean.item() <= valid_mean_min:\n",
        "      print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
        "      valid_mean_min,\n",
        "      val_mean.item()))\n",
        "      torch.save(model.state_dict(), 'pass_predict.pt')\n",
        "      valid_mean_min = val_mean.item()\n",
        "\n",
        "  if early_stopper.early_stop(val_mean.item()):\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xSVWWXOA7l5h",
        "outputId": "e5d2cfdf-7597-4670-effc-7bdef7784983"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "done\n"
          ]
        }
      ],
      "source": [
        "print(\"done\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1g3ZL27p7l5h"
      },
      "outputs": [],
      "source": [
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5437ff28-1eab-4140-ffbc-5e8bde095659",
        "id": "8vk_nOJ27l5h"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.load_state_dict(torch.load('pass_predict.pt'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Akn-7gDw7l5i"
      },
      "outputs": [],
      "source": [
        "itos = vocab.get_itos()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c1a7bd5e-d3d7-4917-9a47-b7b1607c6ec5",
        "id": "LV3xpVuS7l5j"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<sos>'"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "itos[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7a2091d-00ab-4ad2-b583-4dead569e561",
        "id": "6SUPl_px7l5j"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Average BLEU score: 0.719\n"
          ]
        }
      ],
      "source": [
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "\n",
        "model.eval()\n",
        "total_bleu = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off teacher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "\n",
        "        reference = [itos[tok.item()] for tok in trg[1:].reshape(-1)]\n",
        "        total_bleu += sentence_bleu([reference], predicted_sentence)\n",
        "\n",
        "average_bleu = total_bleu / len(test_loader)\n",
        "print(f'Average BLEU score: {average_bleu:.3f}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XONHvGEr7l5k"
      },
      "outputs": [],
      "source": [
        "def predict(model, vocab, device, src_sentence, n=1):\n",
        "    model.eval()\n",
        "\n",
        "    # Convert the source sentence to tokens\n",
        "    src_tokens = [tok for tok in src_sentence]\n",
        "    src_indexes = [vocab[tok] for tok in src_tokens] + [vocab['<eos>']]\n",
        "\n",
        "    # Convert to tensor and add batch dimension\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "\n",
        "    # Initialize target sequence with <sos> token\n",
        "    result = []\n",
        "    for i in range(n):\n",
        "        trg_indexes = [vocab['<sos>']]\n",
        "        predicted_sentence = []\n",
        "        # Predict next token\n",
        "        while True:\n",
        "            trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "            with torch.no_grad():\n",
        "                output = model(src_tensor, trg_tensor)\n",
        "                #print(output)\n",
        "                predicted_token = output.argmax(2)[-1, :].item()\n",
        "                # Check for end of sentence\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                if predicted_token == vocab['<eos>']:\n",
        "                    break\n",
        "\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                # Convert the target sequence to words\n",
        "                predicted_sentence.append(itos[predicted_token])\n",
        "        if ''.join(predicted_sentence[1:]) in result:\n",
        "            continue\n",
        "        result.append(''.join(predicted_sentence[1:]))\n",
        "\n",
        "    return result  # Exclude the <sos> token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc1a5231-a8cf-4d1e-ee00-634291fe0761",
        "id": "SmGtMv_n7l5l"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['aere',\n",
              " 'aels',\n",
              " 'ailer',\n",
              " 'aell',\n",
              " 'ana',\n",
              " 'aie',\n",
              " 'aea',\n",
              " 'ail',\n",
              " 'anc',\n",
              " 'aia',\n",
              " 'aee',\n",
              " 'ain',\n",
              " 'aeon',\n",
              " 'aio',\n",
              " 'ane',\n",
              " 'aear',\n",
              " 'ano',\n",
              " 'aier',\n",
              " 'aeer',\n",
              " 'anar',\n",
              " 'ance',\n",
              " 'aic',\n",
              " 'aer',\n",
              " 'aera',\n",
              " 'ael',\n",
              " 'anan']"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predict(model, vocab, device, \"pyro.archvile@googlemail.com\", 100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a319843d-c432-4f2f-cc58-71395d793554",
        "id": "i7zGcwBf7l5l"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "user,pass\n",
            "bikramojha@yandex.ru,bhp455614\n",
            "adjoa.mansa@yahoo.com,rbsfdplc\n",
            "pyro.archvile@googlemail.com,Omega666\n",
            "red_head76_2000@yahoo.com,0002_67daeh_der\n",
            "laurentnancy@hotmail.fr,27091978zxc\n",
            "19ol64@mail.ru,785333\n",
            "grae123@yahoo.com,enzofan1\n",
            "gulnora_nabiyeva@yahoo.com,1qaz2wsx3edc\n",
            "wydik@ya.ru,525252\n"
          ]
        }
      ],
      "source": [
        "!head -10 /content/drive/MyDrive/Colab/password_predict/data/test.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2506c113-c2f2-4974-ba42-3e44329094f2",
        "id": "vucSa9yr7l5m"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[26, 7, 3, 6, 15, 20, 27, 12, 19, 3, 18, 4, 4, 10, 13, 4, 8]\n",
            "tensor([[[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]], device='cuda:0')\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "src_sentence = \"grae123@yahoo.com\"\n",
        "src_tokens = [tok for tok in src_sentence]\n",
        "src_indexes = [vocab[tok] for tok in src_tokens]\n",
        "print(src_indexes)\n",
        "src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "trg_indexes = [vocab['<sos>']]\n",
        "trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "with torch.no_grad():\n",
        "  output = model(src_tensor, trg_tensor)\n",
        "print(output)\n",
        "output_dim = output.shape[-1]\n",
        "output = output[1:].reshape(-1, output_dim)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8da9c9bd-4ed4-4f85-97f4-e0ec07054b36",
        "id": "fCz961Je7l5o"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predicted_tokens = torch.argmax(output, dim=1)\n",
        "\n",
        "predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "predicted_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vnZLAq_-7l5o"
      },
      "outputs": [],
      "source": [
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off teacher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = ''.join([itos[tok.item()] for tok in predicted_tokens])\n",
        "\n",
        "        reference = ''.join([itos[tok.item()] for tok in trg[1:].reshape(-1)])\n",
        "        print(predicted_sentence)\n",
        "        print(reference)\n",
        "        break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Uz4osjvd7l5p"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-VwznOUD7l5q",
        "outputId": "5033b2cc-29b5-411a-90b3-a75938fa093f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2023 NVIDIA Corporation\n",
            "Built on Tue_Aug_15_22:02:13_PDT_2023\n",
            "Cuda compilation tools, release 12.2, V12.2.140\n",
            "Build cuda_12.2.r12.2/compiler.33191640_0\n"
          ]
        }
      ],
      "source": [
        "!nvcc --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XUYbxtkE7l5q",
        "outputId": "42d38f0f-54dc-41c0-a867-169439d89118"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Python 3.10.12\n"
          ]
        }
      ],
      "source": [
        "!python --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vyvtAXpM7l5r",
        "outputId": "035de6cf-d930-418b-eab1-fe891feeaf66"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fri Dec 22 01:31:12 2023       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  NVIDIA A10                     On  | 00000000:07:00.0 Off |                    0 |\n",
            "|  0%   30C    P8              15W / 150W |      4MiB / 23028MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qfJJnoeN7l5r",
        "outputId": "b0638aad-3da8-4e60-c799-7bb1ad25f07b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "               total        used        free      shared  buff/cache   available\n",
            "Mem:           196Gi       1.0Gi       194Gi       5.0Mi       1.2Gi       194Gi\n",
            "Swap:             0B          0B          0B\n"
          ]
        }
      ],
      "source": [
        "!free -h"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M22p45qQ7l5s",
        "outputId": "162157d2-fede-4579-86d6-09cfd9b639b0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Is available:  True\n",
            "Current Device:  0\n",
            "Pytorch CUDA Compiled version:  12010\n",
            "Pytorch version:  <module 'torch.version' from '/home/ubuntu/.local/lib/python3.10/site-packages/torch/version.py'>\n",
            "pytorch file:  /home/ubuntu/.local/lib/python3.10/site-packages/torch/__init__.py\n"
          ]
        }
      ],
      "source": [
        " !python -c 'import torch ; print(\"Is available: \", torch.cuda.is_available()) ; print(\"Current Device: \", torch.cuda.current_device()) ; print(\"Pytorch CUDA Compiled version: \", torch._C._cuda_getCompiledVersion()) ; print(\"Pytorch version: \", torch.version) ; print(\"pytorch file: \", torch.__file__)'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f937cfa6-2107-4e5b-b923-157434583684",
        "id": "X4MW06HF7l5t"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting fastprogress\n",
            "  Downloading fastprogress-1.0.3-py3-none-any.whl (12 kB)\n",
            "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: fastprogress\n",
            "Successfully installed fastprogress-1.0.3\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install fastprogress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gy50uIDi7l5u",
        "outputId": "b01c8a17-4881-469b-cc17-358955039b47"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Defaulting to user installation because normal site-packages is not writeable\n",
            "Collecting torchtext\n",
            "  Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl.metadata (7.5 kB)\n",
            "Requirement already satisfied: tqdm in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (4.66.1)\n",
            "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (2.31.0)\n",
            "Collecting torch==2.1.2 (from torchtext)\n",
            "  Downloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl.metadata (25 kB)\n",
            "Requirement already satisfied: numpy in /home/ubuntu/.local/lib/python3.10/site-packages (from torchtext) (1.25.2)\n",
            "Collecting torchdata==0.7.1 (from torchtext)\n",
            "  Downloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
            "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.6.0)\n",
            "Requirement already satisfied: typing-extensions in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.1.2->torchtext) (4.8.0)\n",
            "Requirement already satisfied: sympy in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (1.9)\n",
            "Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (2.4)\n",
            "Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch==2.1.2->torchtext) (3.0.3)\n",
            "Collecting fsspec (from torch==2.1.2->torchtext)\n",
            "  Downloading fsspec-2023.12.2-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m121.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m125.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m136.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m55.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m79.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m55.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.1.0 (from torch==2.1.2->torchtext)\n",
            "  Downloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n",
            "Requirement already satisfied: urllib3>=1.25 in /usr/lib/python3/dist-packages (from torchdata==0.7.1->torchtext) (1.26.5)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.2->torchtext)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->torchtext) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->torchtext) (3.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->torchtext) (2020.6.20)\n",
            "Downloading torchtext-0.16.2-cp310-cp310-manylinux1_x86_64.whl (2.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m146.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m106.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m61.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2023.12.2-py3-none-any.whl (168 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m169.0/169.0 kB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m92.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25h\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mInstalling collected packages: triton, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, fsspec, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, torchdata, torchtext\n",
            "Successfully installed fsspec-2023.12.2 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 torch-2.1.2 torchdata-0.7.1 torchtext-0.16.2 triton-2.1.0\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install torchtext"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "23LtlV0D7l5u"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "import torch\n",
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import numpy as np\n",
        "import h5py\n",
        "\n",
        "from fastprogress import master_bar, progress_bar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RcLk6kW17l5v"
      },
      "outputs": [],
      "source": [
        "# Custom char tokenizer\n",
        "def char_tokenizer(text):\n",
        "    return list(text)\n",
        "\n",
        "# Yield tokens from dataset for vocabulary building\n",
        "def yield_tokens(data):\n",
        "    for src, trg in data:\n",
        "        yield char_tokenizer(src)\n",
        "        yield char_tokenizer(trg)\n",
        "\n",
        "# Load data from CSV\n",
        "def load_data_from_csv(filename):\n",
        "    with open(filename, 'r', encoding='utf-8') as file:\n",
        "        csv_reader = csv.reader(file)\n",
        "        data = [(row[0], row[1]) for row in csv_reader]\n",
        "    return data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JTNUYK8V7l5v"
      },
      "outputs": [],
      "source": [
        "train_data_raw = load_data_from_csv('train.csv')\n",
        "val_data_raw = load_data_from_csv('val.csv')\n",
        "test_data_raw = load_data_from_csv('test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dDgpL8pK7l5w",
        "outputId": "42b78dce-ba16-4ca8-9226-f6b808265184"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Vocab length: 4131\n"
          ]
        }
      ],
      "source": [
        "vocab = build_vocab_from_iterator(yield_tokens(train_data_raw), specials=['<sos>', '<eos>', '<unk>'])\n",
        "vocab.set_default_index(vocab[\"<unk>\"])\n",
        "\n",
        "print(f\"Vocab length: {len(vocab)}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "graAk0VJ7l5w"
      },
      "outputs": [],
      "source": [
        "# save vocab\n",
        "import pickle\n",
        "\n",
        "with open('vocab.pkl', 'wb') as f:\n",
        "    pickle.dump(vocab, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tVGpGjjA7l5x"
      },
      "outputs": [],
      "source": [
        "# Dataset class\n",
        "class CSVDataset(Dataset):\n",
        "    def __init__(self, data, vocab):\n",
        "        self.data = data\n",
        "        self.vocab = vocab\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        src, trg = self.data[index]\n",
        "        src = [self.vocab[token] for token in char_tokenizer(src)]\n",
        "        trg = [self.vocab['<sos>']] + [self.vocab[token] for token in char_tokenizer(trg)]  # prepend <sos>\n",
        "        return torch.tensor(src), torch.tensor(trg)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mf7S1vGD7l5x"
      },
      "outputs": [],
      "source": [
        "# Padding function\n",
        "def collate_fn(batch):\n",
        "    src_batch, trg_batch = zip(*batch)\n",
        "    src_batch = pad_sequence(src_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    trg_batch = pad_sequence(trg_batch, padding_value=vocab['<eos>'], batch_first=True)\n",
        "    return src_batch, trg_batch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ONu0Lmiq7l5y"
      },
      "outputs": [],
      "source": [
        "# DataLoader\n",
        "BATCH_SIZE = 512\n",
        "\n",
        "train_dataset = CSVDataset(train_data_raw, vocab)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=True)\n",
        "\n",
        "val_dataset = CSVDataset(val_data_raw, vocab)\n",
        "val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)  # Usually, no need to shuffle val/test data\n",
        "\n",
        "test_dataset = CSVDataset(test_data_raw, vocab)\n",
        "test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d5d34ac5-b797-4c01-f5fe-efd2c816f960",
        "id": "NtEb-Hcc7l5y"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([ 3,  7, 14, 16,  7,  4, 15, 21, 21, 12,  7, 16, 23,  6, 10, 11,  5, 14]) tensor([ 0, 14,  5, 41, 16,  6,  5,  7,  4, 15])\n"
          ]
        }
      ],
      "source": [
        "# check sample from dataset\n",
        "for src, trg in train_dataset:\n",
        "    print(src, trg)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4b5b7fd9-cc29-42f7-99da-ef5fa6b673be",
        "id": "bJCpYRg17l50"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([512, 42]) torch.Size([512, 31])\n"
          ]
        }
      ],
      "source": [
        "# Sample usage:\n",
        "for src, trg in train_loader:\n",
        "    # src and trg here are tokenized, batched, and padded sequences\n",
        "    print(src.shape, trg.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tGNlnSez7l51"
      },
      "outputs": [],
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, src):\n",
        "        embedded = self.dropout(self.embedding(src))\n",
        "        outputs, (hidden, cell) = self.rnn(embedded)\n",
        "        return hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wNmTv-rn7l53"
      },
      "outputs": [],
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, output_dim, emb_dim, hid_dim, n_layers, dropout):\n",
        "        super().__init__()\n",
        "        self.output_dim = output_dim\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)\n",
        "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, input, hidden, cell):\n",
        "        input = input.unsqueeze(0)\n",
        "        embedded = self.dropout(self.embedding(input))\n",
        "        output, (hidden, cell) = self.rnn(embedded, (hidden, cell))\n",
        "        prediction = self.fc_out(output.squeeze(0))\n",
        "        return prediction, hidden, cell"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EillC7mu7l54"
      },
      "outputs": [],
      "source": [
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, encoder, decoder, device):\n",
        "        super().__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.device = device\n",
        "\n",
        "    def forward(self, src, trg, teacher_forcing_ratio=0.5):\n",
        "        trg_len, batch_size = trg.shape\n",
        "        trg_vocab_size = self.decoder.output_dim\n",
        "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
        "        hidden, cell = self.encoder(src)\n",
        "        input = trg[0, :]\n",
        "\n",
        "        for t in range(1, trg_len):\n",
        "            output, hidden, cell = self.decoder(input, hidden, cell)\n",
        "            outputs[t] = output\n",
        "            teacher_force = torch.rand(1) < teacher_forcing_ratio\n",
        "            top1 = output.argmax(1)\n",
        "            input = trg[t] if teacher_force else top1\n",
        "\n",
        "        return outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WKnklyTN7l55"
      },
      "outputs": [],
      "source": [
        "class EarlyStopper:\n",
        "    def __init__(self, patience=1, min_delta=0):\n",
        "        self.patience = patience\n",
        "        self.min_delta = min_delta\n",
        "        self.counter = 0\n",
        "        self.min_validation_loss = np.inf\n",
        "\n",
        "    def early_stop(self, validation_loss):\n",
        "        if validation_loss < self.min_validation_loss:\n",
        "            self.min_validation_loss = validation_loss\n",
        "            self.counter = 0\n",
        "        elif validation_loss > (self.min_validation_loss + self.min_delta):\n",
        "            self.counter += 1\n",
        "            if self.counter >= self.patience:\n",
        "                return True\n",
        "        return False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4_--58Si7l56",
        "outputId": "9051ebdf-11ab-40f1-f3d2-ea6a27e74c39"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        },
        "outputId": "c5f8bc6b-57e6-4af9-f528-85386a889daf",
        "id": "HANivWDl7l56"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
              "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      <progress value='0' class='' max='5' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/5 00:00&lt;?]\n",
              "    </div>\n",
              "    \n",
              "\n",
              "\n",
              "    <div>\n",
              "      <progress value='39495' class='' max='479175' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      8.24% [39495/479175 2:46:54&lt;30:58:07]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from IPython.utils import io\n",
        "\n",
        "# Assuming src and trg are your tokenized, padded source and target sequences\n",
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)\n",
        "\n",
        "lr = 0.001\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
        "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=60, eta_min=1e-5)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "early_stopper = EarlyStopper(patience=10)\n",
        "\n",
        "epochs=5\n",
        "\n",
        "mb = master_bar(range(epochs))\n",
        "mb.names = ['Training loss', 'Validation loss']\n",
        "\n",
        "x = []\n",
        "training_losses = []\n",
        "validation_losses = []\n",
        "\n",
        "valid_mean_min = np.Inf\n",
        "\n",
        "# Training loop\n",
        "for epoch in mb:\n",
        "  x.append(epoch)\n",
        "  with io.capture_output() as captured:\n",
        "      %store x\n",
        "  total_loss = torch.Tensor([0.0]).to(device)\n",
        "\n",
        "  # train\n",
        "  total_train_batches = len(train_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_train_batches\n",
        "  tbc = 0\n",
        "  model.train()\n",
        "  for batch in progress_bar(train_loader, parent=mb):\n",
        "      tbc = tbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store tbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      optimizer.zero_grad()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      total_loss += loss.item()\n",
        "\n",
        "  # decay lr\n",
        "  scheduler.step()\n",
        "  mean = total_loss / len(train_loader)\n",
        "  training_losses.append(mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store training_losses\n",
        "\n",
        "  # validation\n",
        "  model.eval()\n",
        "  total_val_batches = len(val_loader)\n",
        "  with io.capture_output() as captured:\n",
        "      %store total_val_batches\n",
        "  validation_loss = torch.Tensor([0.0]).to(device)\n",
        "  vbc = 0\n",
        "  with torch.no_grad():\n",
        "    for batch in progress_bar(val_loader, parent=mb):\n",
        "      vbc = vbc + 1\n",
        "      with io.capture_output() as captured:\n",
        "          %store vbc\n",
        "      src, trg = batch\n",
        "      src = src.permute(1, 0).to(device).long()\n",
        "      trg = trg.permute(1, 0).to(device).long()\n",
        "      output = model(src, trg)\n",
        "      output_dim = output.shape[-1]\n",
        "      output = output[1:].reshape(-1, output_dim)\n",
        "      trg = trg[1:].reshape(-1)\n",
        "      loss = criterion(output, trg)\n",
        "      validation_loss += loss.item()\n",
        "\n",
        "  val_mean = validation_loss / len(val_loader)\n",
        "  validation_losses.append(val_mean.cpu())\n",
        "  with io.capture_output() as captured:\n",
        "      %store validation_losses\n",
        "  mb.update_graph([[x, training_losses], [x, validation_losses]], [0,epochs])\n",
        "  mb.write(f\"\\nEpoch {epoch}: Training loss {mean.item():.6f} validation loss {val_mean.item():.6f} with lr {lr:.6f}\")\n",
        "\n",
        "  # save model if validation loss has decreased\n",
        "  if val_mean.item() <= valid_mean_min:\n",
        "      print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
        "      valid_mean_min,\n",
        "      val_mean.item()))\n",
        "      torch.save(model.state_dict(), '/content/drive/MyDrive/Colab/password_predict/pass_predict.pt')\n",
        "      valid_mean_min = val_mean.item()\n",
        "\n",
        "  if early_stopper.early_stop(val_mean.item()):\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N9wOtA-w7l57"
      },
      "outputs": [],
      "source": [
        "INPUT_DIM = len(vocab)\n",
        "OUTPUT_DIM = len(vocab)\n",
        "ENC_EMB_DIM = 256\n",
        "DEC_EMB_DIM = 256\n",
        "HID_DIM = 512\n",
        "N_LAYERS = 2\n",
        "ENC_DROPOUT = 0.5\n",
        "DEC_DROPOUT = 0.5\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5437ff28-1eab-4140-ffbc-5e8bde095659",
        "id": "PgHMW0hK7l58"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.load_state_dict(torch.load('/content/drive/MyDrive/Colab/password_predict/pass_predict.pt'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gdy6DWqH7l58"
      },
      "outputs": [],
      "source": [
        "itos = vocab.get_itos()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c1a7bd5e-d3d7-4917-9a47-b7b1607c6ec5",
        "id": "f9coL72d7l58"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<sos>'"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "itos[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7a2091d-00ab-4ad2-b583-4dead569e561",
        "id": "1aRR7GLF7l59"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Average BLEU score: 0.719\n"
          ]
        }
      ],
      "source": [
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "\n",
        "model.eval()\n",
        "total_bleu = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off teacher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "\n",
        "        reference = [itos[tok.item()] for tok in trg[1:].reshape(-1)]\n",
        "        total_bleu += sentence_bleu([reference], predicted_sentence)\n",
        "\n",
        "average_bleu = total_bleu / len(test_loader)\n",
        "print(f'Average BLEU score: {average_bleu:.3f}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AKAvcsxI7l59"
      },
      "outputs": [],
      "source": [
        "def predict(model, vocab, device, src_sentence, n=1):\n",
        "    model.eval()\n",
        "\n",
        "    # Convert the source sentence to tokens\n",
        "    src_tokens = [tok for tok in src_sentence]\n",
        "    src_indexes = [vocab[tok] for tok in src_tokens] + [vocab['<eos>']]\n",
        "\n",
        "    # Convert to tensor and add batch dimension\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "\n",
        "    # Initialize target sequence with <sos> token\n",
        "    result = []\n",
        "    for i in range(n):\n",
        "        trg_indexes = [vocab['<sos>']]\n",
        "        predicted_sentence = []\n",
        "        # Predict next token\n",
        "        while True:\n",
        "            trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "            with torch.no_grad():\n",
        "                output = model(src_tensor, trg_tensor)\n",
        "                #print(output)\n",
        "                predicted_token = output.argmax(2)[-1, :].item()\n",
        "                # Check for end of sentence\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                if predicted_token == vocab['<eos>']:\n",
        "                    break\n",
        "\n",
        "                # Append the predicted token to the target sequence\n",
        "                trg_indexes.append(predicted_token)\n",
        "                # Convert the target sequence to words\n",
        "                predicted_sentence.append(itos[predicted_token])\n",
        "        if ''.join(predicted_sentence[1:]) in result:\n",
        "            continue\n",
        "        result.append(''.join(predicted_sentence[1:]))\n",
        "\n",
        "    return result  # Exclude the <sos> token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc1a5231-a8cf-4d1e-ee00-634291fe0761",
        "id": "F9FCUeKg7l5-"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['aere',\n",
              " 'aels',\n",
              " 'ailer',\n",
              " 'aell',\n",
              " 'ana',\n",
              " 'aie',\n",
              " 'aea',\n",
              " 'ail',\n",
              " 'anc',\n",
              " 'aia',\n",
              " 'aee',\n",
              " 'ain',\n",
              " 'aeon',\n",
              " 'aio',\n",
              " 'ane',\n",
              " 'aear',\n",
              " 'ano',\n",
              " 'aier',\n",
              " 'aeer',\n",
              " 'anar',\n",
              " 'ance',\n",
              " 'aic',\n",
              " 'aer',\n",
              " 'aera',\n",
              " 'ael',\n",
              " 'anan']"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predict(model, vocab, device, \"pyro.archvile@googlemail.com\", 100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a319843d-c432-4f2f-cc58-71395d793554",
        "id": "TdGJecXe7l5_"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "user,pass\n",
            "bikramojha@yandex.ru,bhp455614\n",
            "adjoa.mansa@yahoo.com,rbsfdplc\n",
            "pyro.archvile@googlemail.com,Omega666\n",
            "red_head76_2000@yahoo.com,0002_67daeh_der\n",
            "laurentnancy@hotmail.fr,27091978zxc\n",
            "19ol64@mail.ru,785333\n",
            "grae123@yahoo.com,enzofan1\n",
            "gulnora_nabiyeva@yahoo.com,1qaz2wsx3edc\n",
            "wydik@ya.ru,525252\n"
          ]
        }
      ],
      "source": [
        "!head -10 /content/drive/MyDrive/Colab/password_predict/data/test.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2506c113-c2f2-4974-ba42-3e44329094f2",
        "id": "U-vWsvL87l6A"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[26, 7, 3, 6, 15, 20, 27, 12, 19, 3, 18, 4, 4, 10, 13, 4, 8]\n",
            "tensor([[[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]], device='cuda:0')\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "src_sentence = \"grae123@yahoo.com\"\n",
        "src_tokens = [tok for tok in src_sentence]\n",
        "src_indexes = [vocab[tok] for tok in src_tokens]\n",
        "print(src_indexes)\n",
        "src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
        "trg_indexes = [vocab['<sos>']]\n",
        "trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(1).to(device)\n",
        "with torch.no_grad():\n",
        "  output = model(src_tensor, trg_tensor)\n",
        "print(output)\n",
        "output_dim = output.shape[-1]\n",
        "output = output[1:].reshape(-1, output_dim)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8da9c9bd-4ed4-4f85-97f4-e0ec07054b36",
        "id": "W3KJxwkJ7l6A"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predicted_tokens = torch.argmax(output, dim=1)\n",
        "\n",
        "predicted_sentence = [itos[tok.item()] for tok in predicted_tokens]\n",
        "predicted_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zVVql5Tt7l6B"
      },
      "outputs": [],
      "source": [
        "with torch.no_grad():\n",
        "    for src, trg in test_loader:\n",
        "        src = src.permute(1, 0).to(device).long()\n",
        "        trg = trg.permute(1, 0).to(device).long()\n",
        "\n",
        "        # Turn off teacher forcing\n",
        "        output = model(src, trg, 0)\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output[1:].reshape(-1, output_dim)\n",
        "\n",
        "        # Convert model's outputs to tokens\n",
        "        predicted_tokens = torch.argmax(output, dim=1)\n",
        "        predicted_sentence = ''.join([itos[tok.item()] for tok in predicted_tokens])\n",
        "\n",
        "        reference = ''.join([itos[tok.item()] for tok in trg[1:].reshape(-1)])\n",
        "        print(predicted_sentence)\n",
        "        print(reference)\n",
        "        break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1b_Ti45z7l6B"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}