Transformer_Seq2Seq_fr_en (2).ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "ef9a5eed",
      "metadata": {
        "id": "ef9a5eed"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "import torchtext\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchtext.data.utils import get_tokenizer\n",
        "from collections import Counter\n",
        "from torchtext.vocab import vocab,Vocab\n",
        "from torchtext.utils import download_from_url, extract_archive\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "from torch.utils.data import DataLoader, TensorDataset\n",
        "from torchtext.transforms import ToTensor\n",
        "from torch import Tensor\n",
        "from torch.nn import TransformerEncoder,TransformerDecoder,TransformerEncoderLayer,TransformerDecoderLayer\n",
        "import io\n",
        "import time\n",
        "\n",
        "\n",
        "import random\n",
        "import re\n",
        "import torchtext.data\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import string\n",
        "from torchtext.vocab import FastText as ft\n",
        "from torchtext.data.utils import get_tokenizer\n",
        "from torchtext.vocab import Vectors\n",
        "from torchtext.transforms import ToTensor\n",
        "from torchtext.vocab import Vectors\n",
        "from pathlib import Path\n",
        "import unicodedata"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "id": "61ddfe2b",
      "metadata": {
        "id": "61ddfe2b"
      },
      "outputs": [],
      "source": [
        "import spacy"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ElwoJZL7QYjL"
      },
      "id": "ElwoJZL7QYjL",
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Prepare Data**"
      ],
      "metadata": {
        "id": "j5-Lbh-RQaOF"
      },
      "id": "j5-Lbh-RQaOF"
    },
    {
      "cell_type": "code",
      "source": [
        "class Vocab:\n",
        "    def __init__(self, name):\n",
        "        self.name = name\n",
        "        self.stoi = {}\n",
        "        self.word2count = {}\n",
        "        self.itos = {0: \"<bos>\",1: \"<pad>\", 2:'<eos>',3:'<unk>'}\n",
        "        self.n_words = 4\n",
        "\n",
        "    def add_sentence(self, sentence):\n",
        "        for w in sentence.split(' '):\n",
        "            self.add_word(w)\n",
        "\n",
        "    def add_word(self, w):\n",
        "        if w not in self.stoi:\n",
        "            self.itos[self.n_words] = w\n",
        "            self.stoi = {w:i for (i,w) in self.itos.items()}\n",
        "            self.word2count[w] = 1\n",
        "            self.n_words += 1\n",
        "        else:\n",
        "            self.word2count[w] += 1\n",
        ""
      ],
      "metadata": {
        "id": "szmoB9uk4uhi"
      },
      "id": "szmoB9uk4uhi",
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def unicode_to_ascii(s):\n",
        "    return ''.join(\n",
        "        c for c in unicodedata.normalize('NFD', s)\n",
        "        if unicodedata.category(c) != 'Mn'\n",
        "    )\n",
        "\n",
        "def normalize_string(s):\n",
        "    s = unicode_to_ascii(s.lower().strip())\n",
        "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
        "    s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n",
        "    return s"
      ],
      "metadata": {
        "id": "-Xx52E4I5Aiv"
      },
      "id": "-Xx52E4I5Aiv",
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "%matplotlib inline\n",
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "import sys\n",
        "sys.path.append('../')\n",
        "from fastai.text import *"
      ],
      "metadata": {
        "id": "5I0IEz2B8SeY"
      },
      "id": "5I0IEz2B8SeY",
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "e7PIoYGACVDQ"
      },
      "id": "e7PIoYGACVDQ",
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "df = pd.read_csv('/content/data/questions_easy.csv')"
      ],
      "metadata": {
        "id": "D35tl9iO88He"
      },
      "id": "D35tl9iO88He",
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_RHq64-jAFD1",
        "outputId": "369acf00-f8c4-41e9-9773-f3f6a655ac59"
      },
      "id": "_RHq64-jAFD1",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.loc[df['en'].apply(lambda x: len(x.split(' '))<=20) & df['fr'].apply(lambda x: len(x.split(' '))<=20)]"
      ],
      "metadata": {
        "id": "fWdI3dHb_NvH"
      },
      "id": "fWdI3dHb_NvH",
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "SaI_qMIb_RUn",
        "outputId": "58fcb374-f973-4df8-8789-90a7569d911c"
      },
      "id": "SaI_qMIb_RUn",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  en  \\\n",
              "0                                    what is light ?   \n",
              "1                                       who are we ?   \n",
              "2                           where did we come from ?   \n",
              "3                      what would we do without it ?   \n",
              "4  what is the absolute location latitude and lon...   \n",
              "\n",
              "                                                  fr  \n",
              "0                         qu est ce que la lumiere ?  \n",
              "1                                   ou sommes nous ?  \n",
              "2                                 d ou venons nous ?  \n",
              "3                       que ferions nous sans elle ?  \n",
              "4  quelle sont les coordonnees latitude et longit...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d69c240c-7b56-47d8-b0ca-fc889e8c1dbf\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>en</th>\n",
              "      <th>fr</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>what is light ?</td>\n",
              "      <td>qu est ce que la lumiere ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>who are we ?</td>\n",
              "      <td>ou sommes nous ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>where did we come from ?</td>\n",
              "      <td>d ou venons nous ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>what would we do without it ?</td>\n",
              "      <td>que ferions nous sans elle ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>what is the absolute location latitude and lon...</td>\n",
              "      <td>quelle sont les coordonnees latitude et longit...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d69c240c-7b56-47d8-b0ca-fc889e8c1dbf')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d69c240c-7b56-47d8-b0ca-fc889e8c1dbf button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d69c240c-7b56-47d8-b0ca-fc889e8c1dbf');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2gViFKN8_Rh-"
      },
      "id": "2gViFKN8_Rh-",
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "en_vocab = Vocab('en')\n",
        "fr_vocab = Vocab('fr')"
      ],
      "metadata": {
        "id": "p9nuKBPT_RnL"
      },
      "id": "p9nuKBPT_RnL",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for sent in df['en'].values:\n",
        "    en_vocab.add_sentence(sent)\n",
        "\n",
        "for sent in df['fr'].values:\n",
        "    fr_vocab.add_sentence(sent)"
      ],
      "metadata": {
        "id": "bbyT-G1Y_cyf"
      },
      "id": "bbyT-G1Y_cyf",
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Jq8ntPk3on5r"
      },
      "id": "Jq8ntPk3on5r",
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "en_emb = nn.Embedding(len(en_vocab.stoi),300,padding_idx = 1)\n",
        "wgts = en_emb.weight.data"
      ],
      "metadata": {
        "id": "GBZOW2C__c4b"
      },
      "id": "GBZOW2C__c4b",
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "miss = []\n",
        "for w,i in en_vocab.stoi.items():\n",
        "    try: wgts[i+3] = torch.tensor(vects_en.get_vecs_by_tokens(w))\n",
        "    except: miss.append(w)"
      ],
      "metadata": {
        "id": "YJBFUTEm_c77"
      },
      "id": "YJBFUTEm_c77",
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fr_emb = nn.Embedding(len(fr_vocab.stoi),300,padding_idx=1)\n",
        "wgts = fr_emb.weight.data"
      ],
      "metadata": {
        "id": "1zaI1B3DN_cD"
      },
      "id": "1zaI1B3DN_cD",
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "miss = []\n",
        "for w,i in fr_vocab.stoi.items():\n",
        "    try:wgts[i+1] = torch.tensor(vects_fr.get_vecs_by_tokens(w))\n",
        "    except: miss.append(w)"
      ],
      "metadata": {
        "id": "MSXlWu4iOACw"
      },
      "id": "MSXlWu4iOACw",
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "en_tokenizer = get_tokenizer(None, language='en')\n",
        "fr_tokenizer = get_tokenizer(None, language='fr')"
      ],
      "metadata": {
        "id": "CHTfBWYn_c_v"
      },
      "id": "CHTfBWYn_c_v",
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = [[en_vocab.stoi[y] for y in en_tokenizer(x)] for x in df['en'].values]\n",
        "outputs = [[fr_vocab.stoi[y] for y in fr_tokenizer(x)] for x in df['fr'].values]"
      ],
      "metadata": {
        "id": "7lhvRZiE_dCM"
      },
      "id": "7lhvRZiE_dCM",
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with torch.no_grad():\n",
        "    for i in range(4):\n",
        "        fr_emb.weight[i] = i * torch.ones(300)\n",
        "        en_emb.weight[i] = i * torch.ones(300)"
      ],
      "metadata": {
        "id": "lpdTuTCY_dFe"
      },
      "id": "lpdTuTCY_dFe",
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "QSMof6J8gh8u"
      },
      "id": "QSMof6J8gh8u",
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "THGkQSHnV4T3"
      },
      "execution_count": 26,
      "outputs": [],
      "id": "THGkQSHnV4T3"
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "ab4cb81c"
      },
      "outputs": [],
      "source": [
        "to_tensor = ToTensor(padding_value=1)"
      ],
      "id": "ab4cb81c"
    },
    {
      "cell_type": "code",
      "source": [
        "en_train_data, en_test_data, fr_train_data, fr_test_data = train_test_split(inputs, outputs,test_size=0.2)"
      ],
      "metadata": {
        "id": "FbYOefQcXNvY"
      },
      "id": "FbYOefQcXNvY",
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "en_train_data, en_val_data, fr_train_data, fr_val_data = train_test_split(en_train_data, fr_train_data,test_size=0.2)"
      ],
      "metadata": {
        "id": "bku2WshfY44b"
      },
      "id": "bku2WshfY44b",
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "en_train_data = to_tensor(en_train_data)\n",
        "fr_train_data = to_tensor(fr_train_data)\n",
        "en_val_data = to_tensor(en_val_data)\n",
        "fr_val_data = to_tensor(fr_val_data)\n",
        "en_test_data = to_tensor(en_test_data)\n",
        "fr_test_data = to_tensor(fr_test_data)"
      ],
      "metadata": {
        "id": "aTlgmoYSXOrF"
      },
      "id": "aTlgmoYSXOrF",
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inputs_tensor = ToTensor(padding_value = 1)(inputs)\n",
        "outputs_tensor = ToTensor(padding_value = 1)(outputs)"
      ],
      "metadata": {
        "id": "zRP5R2iw_dIM"
      },
      "id": "zRP5R2iw_dIM",
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inputs_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tVIh41UZCAYs",
        "outputId": "7678e9ef-fc90-4517-be5f-de6946c631f7"
      },
      "id": "tVIh41UZCAYs",
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[   4,    5,    6,  ...,    1,    1,    1],\n",
              "        [   8,    9,   10,  ...,    1,    1,    1],\n",
              "        [  11,   12,   10,  ...,    1,    1,    1],\n",
              "        ...,\n",
              "        [   4,    5,   19,  ...,    1,    1,    1],\n",
              "        [   4,    9,   19,  ...,    1,    1,    1],\n",
              "        [   4,   70, 7119,  ...,    1,    1,    1]])"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ds = TensorDataset(inputs_tensor,outputs_tensor)"
      ],
      "metadata": {
        "id": "R39gdMUZCAbU"
      },
      "id": "R39gdMUZCAbU",
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "5hw5ah4uW1O5"
      },
      "id": "5hw5ah4uW1O5",
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "DhfHx3aACAo3"
      },
      "id": "DhfHx3aACAo3",
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "id": "d5fd7901",
      "metadata": {
        "id": "d5fd7901"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Train Seq2Seq model**"
      ],
      "metadata": {
        "id": "mTL_yQbuQu1_"
      },
      "id": "mTL_yQbuQu1_"
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "id": "72861771",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "72861771",
        "outputId": "a0d8eccd-4016-49ff-dc7f-6d1823038343"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([26588, 20])"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ],
      "source": [
        "en_train_data.size()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "id": "078ada46",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "078ada46",
        "outputId": "6734d8ea-6bbb-47ad-8c7f-1e3ae41f1539"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([26588, 20])"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ],
      "source": [
        "fr_train_data.size()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "id": "5d5338a3",
      "metadata": {
        "id": "5d5338a3"
      },
      "outputs": [],
      "source": [
        "train_ds = TensorDataset(en_train_data,fr_train_data)\n",
        "val_ds = TensorDataset(en_val_data,fr_val_data)\n",
        "test_ds = TensorDataset(en_test_data,fr_test_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "id": "23e2ac36",
      "metadata": {
        "id": "23e2ac36"
      },
      "outputs": [],
      "source": [
        "bs = 128"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "id": "d265c3b7",
      "metadata": {
        "id": "d265c3b7"
      },
      "outputs": [],
      "source": [
        "train_dl = DataLoader(train_ds,batch_size=bs,shuffle=True)\n",
        "val_dl = DataLoader(val_ds,batch_size=bs,shuffle=True)\n",
        "test_dl = DataLoader(test_ds,batch_size=bs,shuffle=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "Fc-c1jam_9af"
      },
      "id": "Fc-c1jam_9af",
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "id": "c98ce9bf",
      "metadata": {
        "id": "c98ce9bf"
      },
      "outputs": [],
      "source": [
        "class Seq2SeqTransformer(nn.Module):\n",
        "    def __init__(self, num_encoder_layers: int, num_decoder_layers: int,\n",
        "                 emb_size: int, src_vocab_size: int, tgt_vocab_size: int,\n",
        "                 dim_feedforward:int = 512, dropout:float = 0.1):\n",
        "        super().__init__()\n",
        "        encoder_layer = TransformerEncoderLayer(d_model=emb_size, nhead=NHEAD,\n",
        "                                                dim_feedforward=dim_feedforward,batch_first=True)\n",
        "        self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers=num_encoder_layers)\n",
        "        decoder_layer = TransformerDecoderLayer(d_model=emb_size, nhead=NHEAD,\n",
        "                                                dim_feedforward=dim_feedforward,batch_first=True)\n",
        "        self.transformer_decoder = TransformerDecoder(decoder_layer, num_layers=num_decoder_layers)\n",
        "\n",
        "        self.generator = nn.Linear(emb_size, tgt_vocab_size)\n",
        "        self.src_tok_emb = TokenEmbedding(src_vocab_size, emb_size)\n",
        "        self.tgt_tok_emb = TokenEmbedding(tgt_vocab_size, emb_size)\n",
        "        self.positional_encoding = PositionalEncoding(emb_size, dropout=dropout)\n",
        "\n",
        "\n",
        "\n",
        "    def forward(self, src: Tensor, trg: Tensor, src_mask: Tensor,\n",
        "                tgt_mask: Tensor, src_padding_mask: Tensor,\n",
        "                tgt_padding_mask: Tensor, memory_key_padding_mask: Tensor):\n",
        "        src_emb = self.positional_encoding(self.src_tok_emb(src))\n",
        "        tgt_emb = self.positional_encoding(self.tgt_tok_emb(trg))\n",
        "        memory = self.transformer_encoder(src_emb, src_mask, src_padding_mask)\n",
        "        outs = self.transformer_decoder(tgt_emb, memory,tgt_mask, None,\n",
        "                                        tgt_padding_mask, memory_key_padding_mask)\n",
        "        return self.generator(outs)\n",
        "\n",
        "    def encode(self, src: Tensor, src_mask: Tensor):\n",
        "        return self.transformer_encoder(self.positional_encoding(\n",
        "                            self.src_tok_emb(src)), src_mask)\n",
        "\n",
        "    def decode(self, tgt: Tensor, memory: Tensor, tgt_mask: Tensor):\n",
        "        return self.transformer_decoder(self.positional_encoding(\n",
        "                          self.tgt_tok_emb(tgt)), memory,\n",
        "                          tgt_mask)\n",
        "\n",
        "class PositionalEncoding(nn.Module):\n",
        "\n",
        "    def __init__(self,emb_size,dropout,max_len=5000):\n",
        "        super().__init__()\n",
        "        den = torch.exp(-torch.arange(0,emb_size,2) * math.log(10000)/emb_size)\n",
        "        pos = torch.arange(0,max_len).reshape(max_len,1)\n",
        "        pos_embedding = torch.zeros((max_len,emb_size))\n",
        "        pos_embedding[:,0::2] = torch.sin(pos * den)\n",
        "        pos_embedding[:,1::2] = torch.cos(pos * den)\n",
        "        pos_embedding = pos_embedding.unsqueeze(-2)\n",
        "\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.register_buffer('pos_embedding',pos_embedding)\n",
        "\n",
        "    def forward(self,token_embedding):\n",
        "        return self.dropout(token_embedding + self.pos_embedding[:token_embedding.size(0),:])\n",
        "\n",
        "\n",
        "\n",
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "id": "4c982438",
      "metadata": {
        "id": "4c982438"
      },
      "outputs": [],
      "source": [
        "class TokenEmbedding(nn.Module):\n",
        "    def __init__(self, vocab_size: int, emb_size):\n",
        "        super(TokenEmbedding, self).__init__()\n",
        "        self.embedding = nn.Embedding(vocab_size, emb_size)\n",
        "        self.emb_size = emb_size\n",
        "    def forward(self, tokens: Tensor):\n",
        "        return self.embedding(tokens.long()) * math.sqrt(self.emb_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "id": "4bb37122",
      "metadata": {
        "id": "4bb37122"
      },
      "outputs": [],
      "source": [
        "def generate_square_subsequent_mask(sz):\n",
        "    mask = (torch.triu(torch.ones((sz,sz),device='cuda') == 1)).transpose(0,1)\n",
        "\n",
        "    mask = mask.float().masked_fill(mask==0,float('-inf')).masked_fill(mask==1,float(0.0))\n",
        "    return mask"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "id": "0f367dc2",
      "metadata": {
        "id": "0f367dc2"
      },
      "outputs": [],
      "source": [
        "def create_mask(src,tgt):\n",
        "    src_seq_len = src.shape[1]\n",
        "    tgt_seq_len = tgt.shape[1]\n",
        "\n",
        "    tgt_mask = generate_square_subsequent_mask(tgt_seq_len)\n",
        "    src_mask = torch.zeros((src_seq_len,src_seq_len),device='cuda').type(torch.bool)\n",
        "\n",
        "    src_padding_mask = (src == 1)\n",
        "    tgt_padding_mask = (tgt == 1)\n",
        "\n",
        "    return src_mask, tgt_mask, src_padding_mask,tgt_padding_mask"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "id": "0a861d45",
      "metadata": {
        "id": "0a861d45"
      },
      "outputs": [],
      "source": [
        "SRC_VOCAB_SIZE = len(en_vocab.stoi)\n",
        "TGT_VOCAB_SIZE = len(fr_vocab.stoi)\n",
        "EMB_SIZE = 512\n",
        "NHEAD = 8\n",
        "FFN_HID_DIM = 512\n",
        "BATCH_SIZE = 128\n",
        "NUM_ENCODER_LAYERS = 3\n",
        "NUM_DECODER_LAYERS = 3\n",
        "NUM_EPOCHS = 50\n",
        "DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "id": "f2fef534",
      "metadata": {
        "id": "f2fef534"
      },
      "outputs": [],
      "source": [
        "transformer = Seq2SeqTransformer(NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS,\n",
        "                                 EMB_SIZE, SRC_VOCAB_SIZE, TGT_VOCAB_SIZE,\n",
        "                                 FFN_HID_DIM)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "id": "06fa325e",
      "metadata": {
        "id": "06fa325e"
      },
      "outputs": [],
      "source": [
        "for p in transformer.parameters():\n",
        "    if p.dim() > 1:\n",
        "        nn.init.xavier_uniform_(p)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "id": "509b62cc",
      "metadata": {
        "id": "509b62cc"
      },
      "outputs": [],
      "source": [
        "transformer = transformer.cuda()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "id": "9865ac7a",
      "metadata": {
        "id": "9865ac7a"
      },
      "outputs": [],
      "source": [
        "loss_fn = torch.nn.CrossEntropyLoss(ignore_index=1)\n",
        "\n",
        "optimizer = torch.optim.Adam(transformer.parameters(),lr=0.0001,betas=(0.9,0.98),eps=1e-9)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "id": "5fb4f880",
      "metadata": {
        "id": "5fb4f880"
      },
      "outputs": [],
      "source": [
        "def train_epoch(model, train_dl, optimizer):\n",
        "    model.train()\n",
        "    losses = 0\n",
        "    for idx, (src, tgt) in enumerate(train_dl):\n",
        "        src = src.cuda()\n",
        "        tgt = tgt.cuda()\n",
        "\n",
        "        tgt_input = tgt[:,:-1]\n",
        "\n",
        "        src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input)\n",
        "\n",
        "        logits = model(src, tgt_input, src_mask,\n",
        "                tgt_mask, src_padding_mask,\n",
        "                tgt_padding_mask, src_padding_mask)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        tgt_out = tgt[:, 1:]\n",
        "        loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1))\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "        losses += loss.item()\n",
        "\n",
        "    return losses / len(train_dl)\n",
        "\n",
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "id": "a6685d63",
      "metadata": {
        "id": "a6685d63"
      },
      "outputs": [],
      "source": [
        "def evaluate(model,val_dl):\n",
        "    model.eval()\n",
        "    losses = 0\n",
        "\n",
        "    for idx, (src,tgt) in enumerate(val_dl):\n",
        "        src = src.cuda()\n",
        "        tgt = tgt.cuda()\n",
        "\n",
        "        tgt_input =tgt[:,:-1]\n",
        "\n",
        "        src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input)\n",
        "\n",
        "        logits = model(src, tgt_input, src_mask,\n",
        "                tgt_mask, src_padding_mask,\n",
        "                tgt_padding_mask, src_padding_mask)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        tgt_out = tgt[:,1:]\n",
        "\n",
        "        loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1))\n",
        "\n",
        "        losses += loss.item()\n",
        "\n",
        "    return losses/len(val_dl)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "id": "93cc0e2e",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "93cc0e2e",
        "outputId": "8b24d17b-d935-4cf8-d334-7526a36a136a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py:4999: UserWarning: Support for mismatched key_padding_mask and attn_mask is deprecated. Use same type for both instead.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0, Train loss: 6.770, Val loss: 5.811, Epoch time = 29.675s \n",
            "Epoch: 1, Train loss: 5.346, Val loss: 4.985, Epoch time = 26.729s \n",
            "Epoch: 2, Train loss: 4.699, Val loss: 4.572, Epoch time = 26.402s \n",
            "Epoch: 3, Train loss: 4.338, Val loss: 4.381, Epoch time = 26.617s \n",
            "Epoch: 4, Train loss: 4.086, Val loss: 4.238, Epoch time = 26.660s \n",
            "Epoch: 5, Train loss: 3.880, Val loss: 4.123, Epoch time = 26.629s \n",
            "Epoch: 6, Train loss: 3.699, Val loss: 4.027, Epoch time = 26.586s \n",
            "Epoch: 7, Train loss: 3.530, Val loss: 3.959, Epoch time = 26.568s \n",
            "Epoch: 8, Train loss: 3.366, Val loss: 3.884, Epoch time = 26.661s \n",
            "Epoch: 9, Train loss: 3.206, Val loss: 3.806, Epoch time = 26.655s \n",
            "Epoch: 10, Train loss: 3.048, Val loss: 3.720, Epoch time = 26.607s \n",
            "Epoch: 11, Train loss: 2.896, Val loss: 3.686, Epoch time = 26.582s \n",
            "Epoch: 12, Train loss: 2.747, Val loss: 3.626, Epoch time = 26.575s \n",
            "Epoch: 13, Train loss: 2.609, Val loss: 3.581, Epoch time = 26.591s \n",
            "Epoch: 14, Train loss: 2.475, Val loss: 3.532, Epoch time = 26.614s \n",
            "Epoch: 15, Train loss: 2.352, Val loss: 3.505, Epoch time = 26.629s \n",
            "Epoch: 16, Train loss: 2.232, Val loss: 3.492, Epoch time = 26.632s \n",
            "Epoch: 17, Train loss: 2.119, Val loss: 3.472, Epoch time = 26.657s \n",
            "Epoch: 18, Train loss: 2.007, Val loss: 3.472, Epoch time = 26.679s \n",
            "Epoch: 19, Train loss: 1.903, Val loss: 3.476, Epoch time = 26.650s \n",
            "Epoch: 20, Train loss: 1.807, Val loss: 3.460, Epoch time = 26.672s \n",
            "Epoch: 21, Train loss: 1.711, Val loss: 3.448, Epoch time = 26.672s \n",
            "Epoch: 22, Train loss: 1.621, Val loss: 3.472, Epoch time = 26.746s \n",
            "Epoch: 23, Train loss: 1.533, Val loss: 3.493, Epoch time = 26.621s \n",
            "Epoch: 24, Train loss: 1.448, Val loss: 3.485, Epoch time = 26.705s \n",
            "Epoch: 25, Train loss: 1.369, Val loss: 3.488, Epoch time = 26.647s \n",
            "Epoch: 26, Train loss: 1.294, Val loss: 3.505, Epoch time = 26.644s \n",
            "Epoch: 27, Train loss: 1.221, Val loss: 3.532, Epoch time = 26.632s \n",
            "Epoch: 28, Train loss: 1.150, Val loss: 3.538, Epoch time = 26.654s \n",
            "Epoch: 29, Train loss: 1.081, Val loss: 3.547, Epoch time = 26.633s \n"
          ]
        }
      ],
      "source": [
        "for epoch in range(30):\n",
        "    start_time = time.time()\n",
        "    train_loss = train_epoch(transformer,train_dl,optimizer)\n",
        "\n",
        "    end_time = time.time()\n",
        "\n",
        "    val_loss = evaluate(transformer,val_dl)\n",
        "\n",
        "    print(f'Epoch: {epoch}, Train loss: {train_loss:.3f}, Val loss: {val_loss:.3f}, '\n",
        "         f'Epoch time = {(end_time - start_time):.3f}s ')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "id": "56dafebb",
      "metadata": {
        "id": "56dafebb"
      },
      "outputs": [],
      "source": [
        "def greedy_decode(model, src, src_mask, max_len, start_symbol):\n",
        "    src = src.cuda()\n",
        "    src_mask = src_mask.cuda()\n",
        "    memory = model.encode(src, src_mask)\n",
        "    ys = torch.ones(1, 1).fill_(2).type(torch.long).cuda()\n",
        "    for i in range(max_len-1):\n",
        "\n",
        "        memory = memory.cuda()\n",
        "        memory_mask = torch.zeros(ys.shape[1], memory.shape[1]).cuda().type(torch.bool)\n",
        "        tgt_mask = (generate_square_subsequent_mask(ys.size(1))\n",
        "                                    .type(torch.bool)).cuda()\n",
        "        out = model.decode(ys, memory, tgt_mask).squeeze(0)[-1]\n",
        "        prob = model.generator(out)\n",
        "        next_word = prob.argmax()\n",
        "        next_word = next_word.item()\n",
        "\n",
        "        ys = torch.cat([ys,\n",
        "                        torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=1)\n",
        "        if next_word == 3:\n",
        "              break\n",
        "    return ys\n",
        "\n",
        "\n",
        "def translate(model, src, src_vocab, tgt_vocab, src_tokenizer):\n",
        "    model.eval()\n",
        "    tokens = [2]+[en_vocab.stoi[x] for x in en_tokenizer(src)]+[3]\n",
        "    num_tokens = len(tokens)\n",
        "    src = (torch.LongTensor(tokens).reshape(1,num_tokens))\n",
        "    src_mask = (torch.zeros(num_tokens, num_tokens)).type(torch.bool)\n",
        "    tgt_tokens = greedy_decode(model, src, src_mask, max_len=num_tokens + 5, start_symbol=2).flatten()\n",
        "    return \" \".join([tgt_vocab.itos[tok] for tok in tgt_tokens.cpu().detach().numpy()]).replace(\"<bos>\", \"\").replace(\"<eos>\", \"\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "v_csBPeNQ3s5"
      },
      "id": "v_csBPeNQ3s5",
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Result**"
      ],
      "metadata": {
        "id": "B9efl-lqQ5xw"
      },
      "id": "B9efl-lqQ5xw"
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "id": "e1cfa32a",
      "metadata": {
        "scrolled": true,
        "id": "e1cfa32a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ed971c08-820e-4e97-d95e-9022d841b8df"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " ? ? ? ? est il dans le groupe de l ile autochtone ? ? ?\n"
          ]
        }
      ],
      "source": [
        "output = translate(transformer, \"what is the major aboriginal group on vancouver island ?\", en_vocab, fr_vocab, en_tokenizer)\n",
        "print(output)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "id": "02970fe8",
      "metadata": {
        "id": "02970fe8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a6618861-0ab7-478f-ba68-4c011cfa751e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " d affaires peut on me rendre au canada ? ? ? ? ? ? ? ? ? ? ? ?\n"
          ]
        }
      ],
      "source": [
        "output1 = translate(transformer, \"who can assist me with questions relating to establishing a business in canada ?\", en_vocab, fr_vocab, en_tokenizer)\n",
        "print(output1)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output2 = translate(transformer, \"where can i find advisories for food drugs medical devices natural health products and consumer products ?\", en_vocab, fr_vocab, en_tokenizer)\n",
        "print(output2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yq20Q0PaFn3_",
        "outputId": "b0c66ae7-55ac-495e-de6b-50de01e8ffce"
      },
      "id": "Yq20Q0PaFn3_",
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " d un aliment special et des produits naturels ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.4"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}