BERT_LSTM_Aug.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XIFwSYH_0KDq",
        "outputId": "cbd2cbda-f763-422a-ab21-125308b7d702"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  data.zip\n",
            "   creating: data/\n",
            "   creating: data/Development/\n",
            "  inflating: data/Development/dev_definitions.txt  \n",
            "  inflating: data/Development/dev_examples.txt  \n",
            "  inflating: data/Development/dev_hypernyms.txt  \n",
            "  inflating: data/Development/dev_labels.txt  \n",
            "  inflating: data/README.txt         \n",
            "   creating: data/Test/\n",
            "  inflating: data/Test/test_definitions.txt  \n",
            "  inflating: data/Test/test_examples.txt  \n",
            "  inflating: data/Test/test_hypernyms.txt  \n",
            "   creating: data/Training/\n",
            "  inflating: data/Training/train_definitions.txt  \n",
            "  inflating: data/Training/train_examples.txt  \n",
            "  inflating: data/Training/train_hypernyms.txt  \n",
            "  inflating: data/Training/train_labels.txt  \n"
          ]
        }
      ],
      "source": [
        "!unzip data.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3QUASp5SZJFX",
        "outputId": "92224e59-911c-4fd2-fbce-36db46ee8086"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tensorflow_text\n",
            "  Downloading tensorflow_text-2.8.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (4.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.9 MB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (0.12.0)\n",
            "Requirement already satisfied: tensorflow<2.9,>=2.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.15.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.2.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.0)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.21.5)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.24.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.17.3)\n",
            "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (13.0.0)\n",
            "Collecting tf-estimator-nightly==2.8.0.dev2021122109\n",
            "  Downloading tf_estimator_nightly-2.8.0.dev2021122109-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 33.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorboard<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.5.3)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.14.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.1.0)\n",
            "Requirement already satisfied: keras<2.9,>=2.8.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (57.4.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.44.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.1.0)\n",
            "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.10.0.2)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.3.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.1.2)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.37.1)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.5.2)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.6.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.35.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.3.6)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.0.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.8.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.2.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.11.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.7.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2.10)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.2.0)\n",
            "Installing collected packages: tf-estimator-nightly, tensorflow-text\n",
            "Successfully installed tensorflow-text-2.8.1 tf-estimator-nightly-2.8.0.dev2021122109\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 7.1 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 37.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 13.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 31.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 3.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.5.1 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.18.0\n"
          ]
        }
      ],
      "source": [
        "!pip install tensorflow_text\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZVe2WcmSCGxy"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import os\n",
        "import errno\n",
        "import random\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn import metrics\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "!rm -rf processed_data\n",
        "\n",
        "try:\n",
        "    os.makedirs(\"processed_data\")\n",
        "except OSError as e:\n",
        "    if e.errno != errno.EEXIST:\n",
        "        raise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eRMB7CMRkvT_"
      },
      "outputs": [],
      "source": [
        "def decontracted(phrase):\n",
        "\n",
        "    phrase = re.sub(r\"won't\", \"will not\", phrase)\n",
        "    phrase = re.sub(r\"can't\", \"can not\", phrase)\n",
        "    phrase = re.sub(r\"gonna\", \"going to\", phrase)\n",
        "    phrase = re.sub(r\"wanna\", \"want to\", phrase)\n",
        "\n",
        "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
        "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
        "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
        "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
        "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
        "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
        "    phrase = re.sub(r\"\\.+\", \" \", phrase)\n",
        "    phrase = re.sub(r\"[^A-Za-z$]\", \" \", phrase)\n",
        "    phrase = re.sub(r\" +\", \" \", phrase)\n",
        "\n",
        "    return phrase"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KWjrzeyqkz0N"
      },
      "outputs": [],
      "source": [
        "def get_df(pref, testing=False):\n",
        "\n",
        "    dataset = []\n",
        "    with open(pref + \"_\" + \"examples.txt\") as examples:\n",
        "        for example in examples:\n",
        "            example = example[:-1].lower().split(\"\\t\")\n",
        "            if example[0] != decontracted(example[0]):\n",
        "                text = re.sub(example[0], \"\\$\", example[2])\n",
        "                text = decontracted(text)\n",
        "                text = re.sub(r\"\\$\", example[0], text)\n",
        "                example[2] = text\n",
        "            else:\n",
        "                example[2] = decontracted(example[2])\n",
        "            dataset.append(example)\n",
        "\n",
        "    with open(pref + \"_\" + \"hypernyms.txt\") as hypernyms:\n",
        "        for index, line in enumerate(hypernyms):\n",
        "            line = line[:-1].lower().split(\"\\t\")\n",
        "            line = \" ; \".join(line)\n",
        "            line = re.sub(r'_', ' ', line)\n",
        "            dataset[index].append(line)\n",
        "\n",
        "    with open(pref + \"_\" + \"definitions.txt\") as definitions:\n",
        "        for index, line in enumerate(definitions):\n",
        "            line = line[:-1].lower().split(\";\")\n",
        "            for i in range(len(line)):\n",
        "                line[i] = line[i].strip()\n",
        "                if dataset[index][0] != decontracted(dataset[index][0]):\n",
        "                    text = re.sub(dataset[index][0], \"\\$\", line[i])\n",
        "                    text = decontracted(text)\n",
        "                    text = re.sub(r\"\\$\", dataset[index][0], text)\n",
        "                    line[i] = text\n",
        "                else:\n",
        "                    line[i] = decontracted(line[i])\n",
        "            line = \" ; \".join(line)\n",
        "            dataset[index].append(line)\n",
        "\n",
        "    cols = [\"target\", \"position\", \"sentence\", \"hypernym\", \"definition\"]\n",
        "\n",
        "    if not testing:\n",
        "        cols.append(\"label\")\n",
        "        with open(pref + \"_\" + \"labels.txt\") as labels:\n",
        "            for index, line in enumerate(labels):\n",
        "                line = line[:-1]\n",
        "                dataset[index].append(line)\n",
        "\n",
        "    df = pd.DataFrame(dataset, columns=cols)\n",
        "\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nUBIiXAck3yc"
      },
      "outputs": [],
      "source": [
        "dev_df = get_df(\"data/Development/dev\")\n",
        "train_df = get_df(\"data/Training/train\")\n",
        "test_df = get_df(\"data/Test/test\", True)\n",
        "\n",
        "dev_df.to_csv(\"processed_data/dev.csv\", index=None)\n",
        "train_df.to_csv(\"processed_data/train.csv\", index=None)\n",
        "test_df.to_csv(\"processed_data/test.csv\", index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IbUPA23KOA9d"
      },
      "outputs": [],
      "source": [
        "def augment(filename):\n",
        "\n",
        "    df = pd.read_csv(filename)\n",
        "\n",
        "    n = df.shape[0]\n",
        "\n",
        "    for i in range(n):\n",
        "        row = df.iloc[i]\n",
        "        if row['label'] == 'T':\n",
        "            if not isinstance(row['hypernym'], str):\n",
        "                continue\n",
        "            position = row['position']\n",
        "            sentence = row['sentence']\n",
        "            hypernyms = row['hypernym'].split(' ; ')\n",
        "            target = row['target']\n",
        "            definition = row['definition']\n",
        "            label = row['label']\n",
        "            for hypernym in hypernyms:\n",
        "                new_row = {'target': hypernym, 'position': position, 'sentence': sentence.replace(target, hypernym), \n",
        "                            'hypernym': target, 'definition': definition, 'label': label}\n",
        "                df = df.append(new_row, ignore_index = True)\n",
        "        \n",
        "        while True:\n",
        "            random_row = df.iloc[random.randint(0, n - 1)]\n",
        "            if random_row['target'] != row['target']:\n",
        "                position = row['position']\n",
        "                sentence = row['sentence']\n",
        "                target = row['target']\n",
        "                hypernyms = random_row['hypernym']\n",
        "                definition = random_row['definition']\n",
        "                label = 'F'\n",
        "                new_row = {'target': target, 'position': position, 'sentence': sentence, \n",
        "                                'hypernym': hypernym, 'definition': definition, 'label': label}\n",
        "                df = df.append(new_row, ignore_index = True)\n",
        "                break\n",
        "\n",
        "    df = shuffle(df)\n",
        "    df.reset_index(drop = True, inplace = True)\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l9TI4QryODhs"
      },
      "outputs": [],
      "source": [
        "aug_train_df = augment('processed_data/train.csv')\n",
        "aug_train_df.to_csv('processed_data/train_aug.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "osPsHkWvIDE8"
      },
      "outputs": [],
      "source": [
        "import tensorflow_text\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from keras import backend as K\n",
        "\n",
        "import transformers\n",
        "transformers.logging.set_verbosity_error()\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from tensorflow.keras.optimizers import Adam"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mjKWllvTIFvZ"
      },
      "outputs": [],
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "transformers.logging.set_verbosity_error()\n",
        "\n",
        "def extract_data(filename, test_data = False):\n",
        "\n",
        "    df = pd.read_csv(filename)\n",
        "    X = {'input_type_ids':[],'input_word_ids':[],'input_mask':[], 'mask1': [], 'mask2': []}\n",
        "    Y = []\n",
        "\n",
        "    for _, row in df.iterrows():\n",
        "\n",
        "        word = row['target']\n",
        "        sentence = row['sentence']\n",
        "        hypernyms = row['hypernym']\n",
        "        definitions = row['definition']\n",
        "\n",
        "        if not test_data:\n",
        "          if row['label'] == 'T':\n",
        "              label = 1.0\n",
        "          else:\n",
        "              label = 0.0\n",
        "\n",
        "        desc = [definitions]\n",
        "\n",
        "        if isinstance(hypernyms, str):\n",
        "          desc.append(hypernyms)\n",
        "        \n",
        "        desc = ' ; '.join(desc)\n",
        "\n",
        "        # sentence = sentence.replace(word.lower(), '$ ' + word + ' $')\n",
        "\n",
        "        x = tokenizer(sentence, desc, max_length = 128, padding='max_length', truncation = True)\n",
        "\n",
        "        sep1 = x['input_ids'].index(102)\n",
        "        sep2 = x['input_ids'].index(102, sep1 + 1)\n",
        "\n",
        "        x['mask1'] = np.zeros((128))\n",
        "        x['mask2'] = np.zeros((128))\n",
        "\n",
        "        # dol1 = x['input_ids'].index(1002)\n",
        "        # dol2 = x['input_ids'].index(1002, dol1 + 1)\n",
        "\n",
        "        for i in range(1, sep1):\n",
        "            x['mask1'][i] = 1.0\n",
        "        \n",
        "        for i in range(sep1 + 1, sep2):\n",
        "            x['mask2'][i] = 1.0\n",
        "\n",
        "        # for i in range(dol1, dol2 + 1):\n",
        "        #     x['token_type_ids'][i] = 1.0\n",
        "        #     x['mask2'][i] = 1.0\n",
        "\n",
        "        X['input_type_ids'].append(x['token_type_ids'])\n",
        "        X['input_word_ids'].append(x['input_ids'])\n",
        "        X['input_mask'].append(x['attention_mask'])\n",
        "        X['mask1'].append(x['mask1'])\n",
        "        X['mask2'].append(x['mask2'])\n",
        "\n",
        "        if not test_data:\n",
        "          Y.append(label)     \n",
        "\n",
        "    X['input_type_ids'] = np.array(X['input_type_ids'], dtype=np.float32)\n",
        "    X['input_word_ids'] = np.array(X['input_word_ids'], dtype=np.float32)\n",
        "    X['input_mask'] = np.array(X['input_mask'], dtype=np.float32)\n",
        "\n",
        "    X['mask1'] = np.array(X['mask1'], dtype=np.float32)\n",
        "    X['mask2'] = np.array(X['mask2'], dtype=np.float32)\n",
        "\n",
        "    Y = np.array(Y)\n",
        "\n",
        "    return X, Y\n",
        "\n",
        "X,Y = extract_data('processed_data/train_aug.csv')\n",
        "Xval, Yval = extract_data('processed_data/dev.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmJ4l6L2YIz7",
        "outputId": "d523e7e3-b631-4824-c629-6dfdc0facc8d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_word_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_mask (InputLayer)        [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_type_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_3 (InputLayer)           [(None, 128, 1)]     0           []                               \n",
            "                                                                                                  \n",
            " bert-encoder (KerasLayer)      [(None, 768),        109482241   ['input_word_ids[0][0]',         \n",
            "                                 (None, 128, 768)]                'input_mask[0][0]',             \n",
            "                                                                  'input_type_ids[0][0]']         \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 128, 1)]     0           []                               \n",
            "                                                                                                  \n",
            " tf.math.multiply_2 (TFOpLambda  (None, 128, 768)    0           ['input_3[0][0]',                \n",
            " )                                                                'bert-encoder[1][1]']           \n",
            "                                                                                                  \n",
            " tf.math.multiply_3 (TFOpLambda  (None, 128, 768)    0           ['input_4[0][0]',                \n",
            " )                                                                'bert-encoder[1][1]']           \n",
            "                                                                                                  \n",
            " masking_2 (Masking)            (None, 128, 768)     0           ['tf.math.multiply_2[1][0]']     \n",
            "                                                                                                  \n",
            " masking_3 (Masking)            (None, 128, 768)     0           ['tf.math.multiply_3[1][0]']     \n",
            "                                                                                                  \n",
            " lstm_2 (LSTM)                  (None, 32)           102528      ['masking_2[1][0]']              \n",
            "                                                                                                  \n",
            " lstm_3 (LSTM)                  (None, 32)           102528      ['masking_3[1][0]']              \n",
            "                                                                                                  \n",
            " dense_1 (Dense)                (None, 32)           24608       ['bert-encoder[1][0]']           \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 96)           0           ['lstm_2[1][0]',                 \n",
            "                                                                  'lstm_3[1][0]',                 \n",
            "                                                                  'dense_1[1][0]']                \n",
            "                                                                                                  \n",
            " dropout_1 (Dropout)            (None, 96)           0           ['concatenate[1][0]']            \n",
            "                                                                                                  \n",
            " output (Dense)                 (None, 1)            97          ['dropout_1[1][0]']              \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,712,002\n",
            "Trainable params: 109,712,001\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Input layers -----------------------------\n",
        "\n",
        "input_type_ids = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_type_ids\")\n",
        "input_word_ids = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_word_ids\")\n",
        "input_mask = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_mask\")\n",
        "\n",
        "input_sent_mask = tf.keras.layers.Input(shape=(128), dtype = tf.float32, name = \"input_sent_mask\")\n",
        "input_desc_mask = tf.keras.layers.Input(shape=(128), dtype = tf.float32, name = \"input_desc_mask\")\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "\n",
        "# Encoder -----------------------------------\n",
        "\n",
        "encoder = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2\", name = \"bert-encoder\", trainable = True)\n",
        "pooled_output, sequence_output = encoder([input_word_ids, input_mask, input_type_ids])\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "\n",
        "# Decoder -----------------------------------\n",
        "\n",
        "input_sent_mask = tf.expand_dims(input_sent_mask, axis = 2)\n",
        "sent_output = tf.multiply(input_sent_mask, sequence_output)\n",
        "sent_output = tf.keras.layers.Masking(mask_value = 0)(sent_output)\n",
        "sent_output = tf.keras.layers.LSTM(32, activation = 'relu')(sent_output)\n",
        "\n",
        "input_desc_mask = tf.expand_dims(input_desc_mask, axis = 2)\n",
        "desc_output = tf.multiply(input_desc_mask, sequence_output)\n",
        "desc_output = tf.keras.layers.Masking(mask_value = 0)(desc_output)\n",
        "desc_output = tf.keras.layers.LSTM(32, activation = 'relu')(desc_output)\n",
        "\n",
        "pooled_output = tf.keras.layers.Dense(32, activation = 'relu')(pooled_output)\n",
        "\n",
        "concat = tf.keras.layers.Concatenate(name = \"concatenate\")([sent_output, desc_output, pooled_output])\n",
        "concat = tf.keras.layers.Dropout(0.3)(concat)\n",
        "output = tf.keras.layers.Dense(1, activation = 'sigmoid', name = \"output\")(concat)\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "# Encoder-Decoder Model ---------------------\n",
        "\n",
        "model = tf.keras.models.Model(\n",
        "      inputs = [\n",
        "        input_word_ids,\n",
        "        input_mask,\n",
        "        input_type_ids,\n",
        "        input_sent_mask,\n",
        "        input_desc_mask], \n",
        "      outputs = output)\n",
        "\n",
        "model.compile(optimizer=Adam(learning_rate = 1e-6), loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GCEGsTN7lXpT"
      },
      "outputs": [],
      "source": [
        "def find_optimal_threshold(target, predicted):\n",
        "    fpr, tpr, thresholds = metrics.roc_curve(target, predicted)\n",
        "    fpr = fpr[1:]\n",
        "    tpr = tpr[1:]\n",
        "    thresholds = thresholds[1:]\n",
        "    x = tpr - fpr\n",
        "    idx = np.argmax(x)\n",
        "    return thresholds[idx]\n",
        "\n",
        "class roc_auc_threshold(tf.keras.callbacks.Callback):\n",
        "\n",
        "    def __init__(self, training_data, validation_data):\n",
        "        self.x = training_data[0]\n",
        "        self.y = training_data[1]\n",
        "        self.x_val = validation_data[0]\n",
        "        self.y_val = validation_data[1]\n",
        "\n",
        "    def on_train_begin(self, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_train_end(self, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_epoch_begin(self, epoch, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs={}):\n",
        "        model.save('model.' + str(epoch) + '.h5')\n",
        "        y_pred_val = self.model.predict(self.x_val)\n",
        "        optimal_threshold = find_optimal_threshold(self.y_val, y_pred_val)\n",
        "        y_pred_val = y_pred_val.flatten()\n",
        "        y_truth_val = self.y_val.flatten()\n",
        "        val_correct = 0\n",
        "        val_total = len(y_pred_val)\n",
        "        for i in range(len(y_pred_val)):\n",
        "          pred = 0\n",
        "          if y_pred_val[i] >= optimal_threshold:\n",
        "            pred = 1\n",
        "          if pred == y_truth_val[i]:\n",
        "            val_correct += 1\n",
        "        print(' optimal_threshold:', round(optimal_threshold, 5), '-',  \n",
        "              'val_accuracy_optimal:', round(val_correct / val_total, 5))\n",
        "        return\n",
        "\n",
        "    def on_batch_begin(self, batch, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_batch_end(self, batch, logs={}):\n",
        "        return\n",
        "\n",
        "cb = roc_auc_threshold(training_data = ([X['input_word_ids'], X['input_mask'], X['input_type_ids'], X['mask1'], X['mask2']], Y),\n",
        "                  validation_data = ([Xval['input_word_ids'], Xval['input_mask'], Xval['input_type_ids'], Xval['mask1'], Xval['mask2']], Yval))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ySbknbfzYmVf",
        "outputId": "797b7b2c-f97e-4cd8-aee4-2e1c1b65520d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.6461 - accuracy: 0.6206 optimal_threshold: 0.63818 - val_accuracy_optimal: 0.65039\n",
            "433/433 [==============================] - 759s 2s/step - loss: 0.6461 - accuracy: 0.6206 - val_loss: 0.6516 - val_accuracy: 0.6452\n",
            "Epoch 2/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.5060 - accuracy: 0.7779 optimal_threshold: 0.55713 - val_accuracy_optimal: 0.7018\n",
            "433/433 [==============================] - 737s 2s/step - loss: 0.5060 - accuracy: 0.7779 - val_loss: 0.6291 - val_accuracy: 0.6889\n",
            "Epoch 3/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.4002 - accuracy: 0.8364 optimal_threshold: 0.73891 - val_accuracy_optimal: 0.71722\n",
            "433/433 [==============================] - 737s 2s/step - loss: 0.4002 - accuracy: 0.8364 - val_loss: 0.6356 - val_accuracy: 0.6967\n",
            "Epoch 4/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.3330 - accuracy: 0.8686 optimal_threshold: 0.76924 - val_accuracy_optimal: 0.74036\n",
            "433/433 [==============================] - 741s 2s/step - loss: 0.3330 - accuracy: 0.8686 - val_loss: 0.6787 - val_accuracy: 0.7095\n"
          ]
        }
      ],
      "source": [
        "history = model.fit([X['input_word_ids'], X['input_mask'], X['input_type_ids'], X['mask1'], X['mask2']], Y, \n",
        "          epochs = 4, batch_size = 16, callbacks=[cb], verbose = 1,\n",
        "          validation_data = ([Xval['input_word_ids'], Xval['input_mask'], Xval['input_type_ids'], Xval['mask1'], Xval['mask2']], Yval))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ckgwg9lXO2X4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "12c4be56-9878-42da-94f5-4f637daf0b68"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        }
      ],
      "source": [
        "epoch = 2\n",
        "pretrained_model = tf.keras.models.load_model('model.' + str(epoch) + '.h5', custom_objects={'KerasLayer': hub.KerasLayer})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rA6k-oGg66nU"
      },
      "outputs": [],
      "source": [
        "Xtest, Ytest = extract_data('processed_data/test.csv', True)\n",
        "y_predicted = pretrained_model.predict([Xtest['input_word_ids'], Xtest['input_mask'], Xtest['input_type_ids'], Xtest['mask1'], Xtest['mask2']])\n",
        "y_predicted = y_predicted.flatten()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimal_threshold = 0.73891\n",
        "\n",
        "file = open('all_output.txt', 'w')\n",
        "for val in y_predicted:\n",
        "  if val < optimal_threshold:\n",
        "    print('F',file=file)\n",
        "  else:\n",
        "    print('T',file=file)\n",
        "file.close()"
      ],
      "metadata": {
        "id": "4cVE5umM-mAg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "anv-FXthURH9"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "BERT + LSTM + Aug.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}