BERT_LSTM_WE.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XIFwSYH_0KDq",
        "outputId": "afc33805-30ee-414a-be16-acb5ad73b6d8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  data.zip\n",
            "   creating: data/\n",
            "   creating: data/Development/\n",
            "  inflating: data/Development/dev_definitions.txt  \n",
            "  inflating: data/Development/dev_examples.txt  \n",
            "  inflating: data/Development/dev_hypernyms.txt  \n",
            "  inflating: data/Development/dev_labels.txt  \n",
            "  inflating: data/README.txt         \n",
            "   creating: data/Test/\n",
            "  inflating: data/Test/test_definitions.txt  \n",
            "  inflating: data/Test/test_examples.txt  \n",
            "  inflating: data/Test/test_hypernyms.txt  \n",
            "   creating: data/Training/\n",
            "  inflating: data/Training/train_definitions.txt  \n",
            "  inflating: data/Training/train_examples.txt  \n",
            "  inflating: data/Training/train_hypernyms.txt  \n",
            "  inflating: data/Training/train_labels.txt  \n"
          ]
        }
      ],
      "source": [
        "!unzip data.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3QUASp5SZJFX",
        "outputId": "9d671f2c-83bc-449f-edc6-0ee502ad13dc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tensorflow_text\n",
            "  Downloading tensorflow_text-2.8.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (4.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.9 MB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow<2.9,>=2.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (0.12.0)\n",
            "Requirement already satisfied: keras<2.9,>=2.8.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.24.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.1.0)\n",
            "Requirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.5.3)\n",
            "Requirement already satisfied: tensorboard<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (57.4.0)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.14.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.1.2)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.6.3)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.1.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.44.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.17.3)\n",
            "Collecting tf-estimator-nightly==2.8.0.dev2021122109\n",
            "  Downloading tf_estimator_nightly-2.8.0.dev2021122109-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 46.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.0)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.21.5)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.3.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.2.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.15.0)\n",
            "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (13.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.10.0.2)\n",
            "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.0.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.37.1)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.5.2)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.8.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.3.6)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.4.6)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.6.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.35.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.0.1)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.2.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.8)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.11.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.7.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2.10)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.2.0)\n",
            "Installing collected packages: tf-estimator-nightly, tensorflow-text\n",
            "Successfully installed tensorflow-text-2.8.1 tf-estimator-nightly-2.8.0.dev2021122109\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 5.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 6.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 44.2 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 23.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 39.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.5.1 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.18.0\n"
          ]
        }
      ],
      "source": [
        "!pip install tensorflow_text\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZVe2WcmSCGxy"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import os\n",
        "import errno\n",
        "import random\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn import metrics\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "!rm -rf processed_data\n",
        "\n",
        "try:\n",
        "    os.makedirs(\"processed_data\")\n",
        "except OSError as e:\n",
        "    if e.errno != errno.EEXIST:\n",
        "        raise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eRMB7CMRkvT_"
      },
      "outputs": [],
      "source": [
        "def decontracted(phrase):\n",
        "\n",
        "    phrase = re.sub(r\"won't\", \"will not\", phrase)\n",
        "    phrase = re.sub(r\"can't\", \"can not\", phrase)\n",
        "    phrase = re.sub(r\"gonna\", \"going to\", phrase)\n",
        "    phrase = re.sub(r\"wanna\", \"want to\", phrase)\n",
        "\n",
        "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
        "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
        "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
        "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
        "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
        "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
        "    phrase = re.sub(r\"\\.+\", \" \", phrase)\n",
        "    phrase = re.sub(r\"[^A-Za-z$]\", \" \", phrase)\n",
        "    phrase = re.sub(r\" +\", \" \", phrase)\n",
        "\n",
        "    return phrase"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KWjrzeyqkz0N"
      },
      "outputs": [],
      "source": [
        "def get_df(pref, testing=False):\n",
        "\n",
        "    dataset = []\n",
        "    with open(pref + \"_\" + \"examples.txt\") as examples:\n",
        "        for example in examples:\n",
        "            example = example[:-1].lower().split(\"\\t\")\n",
        "            if example[0] != decontracted(example[0]):\n",
        "                text = re.sub(example[0], \"\\$\", example[2])\n",
        "                text = decontracted(text)\n",
        "                text = re.sub(r\"\\$\", example[0], text)\n",
        "                example[2] = text\n",
        "            else:\n",
        "                example[2] = decontracted(example[2])\n",
        "            dataset.append(example)\n",
        "\n",
        "    with open(pref + \"_\" + \"hypernyms.txt\") as hypernyms:\n",
        "        for index, line in enumerate(hypernyms):\n",
        "            line = line[:-1].lower().split(\"\\t\")\n",
        "            line = \" ; \".join(line)\n",
        "            line = re.sub(r'_', ' ', line)\n",
        "            dataset[index].append(line)\n",
        "\n",
        "    with open(pref + \"_\" + \"definitions.txt\") as definitions:\n",
        "        for index, line in enumerate(definitions):\n",
        "            line = line[:-1].lower().split(\";\")\n",
        "            for i in range(len(line)):\n",
        "                line[i] = line[i].strip()\n",
        "                if dataset[index][0] != decontracted(dataset[index][0]):\n",
        "                    text = re.sub(dataset[index][0], \"\\$\", line[i])\n",
        "                    text = decontracted(text)\n",
        "                    text = re.sub(r\"\\$\", dataset[index][0], text)\n",
        "                    line[i] = text\n",
        "                else:\n",
        "                    line[i] = decontracted(line[i])\n",
        "            line = \" ; \".join(line)\n",
        "            dataset[index].append(line)\n",
        "\n",
        "    cols = [\"target\", \"position\", \"sentence\", \"hypernym\", \"definition\"]\n",
        "\n",
        "    if not testing:\n",
        "        cols.append(\"label\")\n",
        "        with open(pref + \"_\" + \"labels.txt\") as labels:\n",
        "            for index, line in enumerate(labels):\n",
        "                line = line[:-1]\n",
        "                dataset[index].append(line)\n",
        "\n",
        "    df = pd.DataFrame(dataset, columns=cols)\n",
        "\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nUBIiXAck3yc"
      },
      "outputs": [],
      "source": [
        "dev_df = get_df(\"data/Development/dev\")\n",
        "train_df = get_df(\"data/Training/train\")\n",
        "test_df = get_df(\"data/Test/test\", True)\n",
        "\n",
        "dev_df.to_csv(\"processed_data/dev.csv\", index=None)\n",
        "train_df.to_csv(\"processed_data/train.csv\", index=None)\n",
        "test_df.to_csv(\"processed_data/test.csv\", index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IbUPA23KOA9d"
      },
      "outputs": [],
      "source": [
        "def augment(filename):\n",
        "\n",
        "    df = pd.read_csv(filename)\n",
        "\n",
        "    n = df.shape[0]\n",
        "\n",
        "    for i in range(n):\n",
        "        row = df.iloc[i]\n",
        "        if row['label'] == 'T':\n",
        "            if not isinstance(row['hypernym'], str):\n",
        "                continue\n",
        "            position = row['position']\n",
        "            sentence = row['sentence']\n",
        "            hypernyms = row['hypernym'].split(' ; ')\n",
        "            target = row['target']\n",
        "            definition = row['definition']\n",
        "            label = row['label']\n",
        "            for hypernym in hypernyms:\n",
        "                new_row = {'target': hypernym, 'position': position, 'sentence': sentence.replace(target, hypernym), \n",
        "                            'hypernym': target, 'definition': definition, 'label': label}\n",
        "                df = df.append(new_row, ignore_index = True)\n",
        "        \n",
        "        while True:\n",
        "            random_row = df.iloc[random.randint(0, n - 1)]\n",
        "            if random_row['target'] != row['target']:\n",
        "                position = row['position']\n",
        "                sentence = row['sentence']\n",
        "                target = row['target']\n",
        "                hypernyms = random_row['hypernym']\n",
        "                definition = random_row['definition']\n",
        "                label = 'F'\n",
        "                new_row = {'target': target, 'position': position, 'sentence': sentence, \n",
        "                                'hypernym': hypernym, 'definition': definition, 'label': label}\n",
        "                df = df.append(new_row, ignore_index = True)\n",
        "                break\n",
        "\n",
        "    df = shuffle(df)\n",
        "    df.reset_index(drop = True, inplace = True)\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l9TI4QryODhs"
      },
      "outputs": [],
      "source": [
        "aug_train_df = augment('processed_data/train.csv')\n",
        "aug_train_df.to_csv('processed_data/train_aug.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "osPsHkWvIDE8"
      },
      "outputs": [],
      "source": [
        "import tensorflow_text\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from keras import backend as K\n",
        "\n",
        "import transformers\n",
        "transformers.logging.set_verbosity_error()\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from tensorflow.keras.optimizers import Adam"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mjKWllvTIFvZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "8aee578f0b8d4936b2f85d81b928acd4",
            "6d032aa9257a487fafe22b164326f393",
            "3b1b01acf06a4f1aa85f266a742263ab",
            "9ef5c0713cc341329ad0b9c08340215c",
            "8209ede274ec45308e6ab6d6c4ae1b05",
            "8190d772889240f587bbb38bdcfd98ab",
            "87fdc6a90e3141dc8a257203d3c5fc58",
            "ab99601e5df34514a2af1c5433bb4e17",
            "4ac65c9de5fa4861841c6b38a8a3afd7",
            "82745ea90a2a458c89d2dff33c324f1f",
            "03ee137cd12c463fbf4c9b51807da661",
            "48e627da295d48a499ea5f909a6074d9",
            "d7c0b325456c46ebaf75890d0d79083a",
            "23ec4e0d866f4777a38cb2ade39f166e",
            "7f4c5a0e3ba74acfb5c463faf1f9289a",
            "f9cbb24742534149ac161af87a3ea0c4",
            "0c1e798f6c7346bb8f56fd1894c5a618",
            "066892a85d2243e4bed654ef3313c080",
            "7634b4709e5f49c9a5913e5c54dda1b4",
            "c750ebbdc25245f88e61c19a2d767a2b",
            "71e2c4404383414d8813208d6001a02c",
            "c327504bfb11482da748d6758c442e9a",
            "7711362bff1f4efaae10e5edacf0c84c",
            "4242ea68fa814bb59ec0f34a739e013d",
            "0efc6092f0cf42bb87f25f0990e921c7",
            "7ef871487a7747f18120209728034fd6",
            "6eb2fe1e0eb94c54ba0ffd1834edba0b",
            "7b6c00dcaf644594b4ea5d21037adef1",
            "72ef53eb3f5c499d927b137aa06e2620",
            "3e2d4a5080c64503bee3f7966cba3ddb",
            "b916ceb0eff24a2893d1090a5f2dfc0c",
            "21c5255a8c564f73ae45d620f5284867",
            "1b64b483873842d3bd170b1640e5a5ba"
          ]
        },
        "outputId": "46fd5855-cdf4-44be-bf0f-62668aec6ec1"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8aee578f0b8d4936b2f85d81b928acd4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "48e627da295d48a499ea5f909a6074d9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7711362bff1f4efaae10e5edacf0c84c"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "transformers.logging.set_verbosity_error()\n",
        "\n",
        "def extract_data(filename, test_data = False):\n",
        "\n",
        "    df = pd.read_csv(filename)\n",
        "    X = {'input_type_ids':[],'input_word_ids':[],'input_mask':[], 'mask1': [], 'mask2': []}\n",
        "    Y = []\n",
        "\n",
        "    for _, row in df.iterrows():\n",
        "\n",
        "        word = row['target']\n",
        "        sentence = row['sentence']\n",
        "        hypernyms = row['hypernym']\n",
        "        definitions = row['definition']\n",
        "\n",
        "        if not test_data:\n",
        "          if row['label'] == 'T':\n",
        "              label = 1.0\n",
        "          else:\n",
        "              label = 0.0\n",
        "\n",
        "        desc = [definitions]\n",
        "\n",
        "        if isinstance(hypernyms, str):\n",
        "          desc.append(hypernyms)\n",
        "        \n",
        "        desc = ' ; '.join(desc)\n",
        "\n",
        "        sentence = sentence.replace(word.lower(), '$ ' + word + ' $')\n",
        "\n",
        "        x = tokenizer(sentence, desc, max_length = 128, padding='max_length', truncation = True)\n",
        "\n",
        "        sep1 = x['input_ids'].index(102)\n",
        "        sep2 = x['input_ids'].index(102, sep1 + 1)\n",
        "\n",
        "        x['mask1'] = np.zeros((128))\n",
        "        x['mask2'] = np.zeros((128))\n",
        "\n",
        "        dol1 = x['input_ids'].index(1002)\n",
        "        dol2 = x['input_ids'].index(1002, dol1 + 1)\n",
        "\n",
        "        for i in range(1, sep1):\n",
        "            x['mask1'][i] = 1.0\n",
        "        \n",
        "        for i in range(sep1 + 1, sep2):\n",
        "            x['mask2'][i] = 1.0\n",
        "\n",
        "        for i in range(dol1, dol2 + 1):\n",
        "            x['token_type_ids'][i] = 1.0\n",
        "            x['mask2'][i] = 1.0\n",
        "\n",
        "        X['input_type_ids'].append(x['token_type_ids'])\n",
        "        X['input_word_ids'].append(x['input_ids'])\n",
        "        X['input_mask'].append(x['attention_mask'])\n",
        "        X['mask1'].append(x['mask1'])\n",
        "        X['mask2'].append(x['mask2'])\n",
        "\n",
        "        if not test_data:\n",
        "          Y.append(label)     \n",
        "\n",
        "    X['input_type_ids'] = np.array(X['input_type_ids'], dtype=np.float32)\n",
        "    X['input_word_ids'] = np.array(X['input_word_ids'], dtype=np.float32)\n",
        "    X['input_mask'] = np.array(X['input_mask'], dtype=np.float32)\n",
        "\n",
        "    X['mask1'] = np.array(X['mask1'], dtype=np.float32)\n",
        "    X['mask2'] = np.array(X['mask2'], dtype=np.float32)\n",
        "\n",
        "    Y = np.array(Y)\n",
        "\n",
        "    return X, Y\n",
        "\n",
        "X,Y = extract_data('processed_data/train_aug.csv')\n",
        "Xval, Yval = extract_data('processed_data/dev.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmJ4l6L2YIz7",
        "outputId": "3b5477f3-6fce-436d-af79-9da9be377fe6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_word_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_mask (InputLayer)        [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_type_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_3 (InputLayer)           [(None, 128, 1)]     0           []                               \n",
            "                                                                                                  \n",
            " bert-encoder (KerasLayer)      [(None, 768),        109482241   ['input_word_ids[0][0]',         \n",
            "                                 (None, 128, 768)]                'input_mask[0][0]',             \n",
            "                                                                  'input_type_ids[0][0]']         \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 128, 1)]     0           []                               \n",
            "                                                                                                  \n",
            " tf.math.multiply_2 (TFOpLambda  (None, 128, 768)    0           ['input_3[0][0]',                \n",
            " )                                                                'bert-encoder[1][1]']           \n",
            "                                                                                                  \n",
            " tf.math.multiply_3 (TFOpLambda  (None, 128, 768)    0           ['input_4[0][0]',                \n",
            " )                                                                'bert-encoder[1][1]']           \n",
            "                                                                                                  \n",
            " masking_2 (Masking)            (None, 128, 768)     0           ['tf.math.multiply_2[1][0]']     \n",
            "                                                                                                  \n",
            " masking_3 (Masking)            (None, 128, 768)     0           ['tf.math.multiply_3[1][0]']     \n",
            "                                                                                                  \n",
            " lstm_2 (LSTM)                  (None, 32)           102528      ['masking_2[1][0]']              \n",
            "                                                                                                  \n",
            " lstm_3 (LSTM)                  (None, 32)           102528      ['masking_3[1][0]']              \n",
            "                                                                                                  \n",
            " dense_1 (Dense)                (None, 32)           24608       ['bert-encoder[1][0]']           \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 96)           0           ['lstm_2[1][0]',                 \n",
            "                                                                  'lstm_3[1][0]',                 \n",
            "                                                                  'dense_1[1][0]']                \n",
            "                                                                                                  \n",
            " dropout_1 (Dropout)            (None, 96)           0           ['concatenate[1][0]']            \n",
            "                                                                                                  \n",
            " output (Dense)                 (None, 1)            97          ['dropout_1[1][0]']              \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,712,002\n",
            "Trainable params: 109,712,001\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Input layers -----------------------------\n",
        "\n",
        "input_type_ids = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_type_ids\")\n",
        "input_word_ids = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_word_ids\")\n",
        "input_mask = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_mask\")\n",
        "\n",
        "input_sent_mask = tf.keras.layers.Input(shape=(128), dtype = tf.float32, name = \"input_sent_mask\")\n",
        "input_desc_mask = tf.keras.layers.Input(shape=(128), dtype = tf.float32, name = \"input_desc_mask\")\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "\n",
        "# Encoder -----------------------------------\n",
        "\n",
        "encoder = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2\", name = \"bert-encoder\", trainable = True)\n",
        "pooled_output, sequence_output = encoder([input_word_ids, input_mask, input_type_ids])\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "\n",
        "# Decoder -----------------------------------\n",
        "\n",
        "input_sent_mask = tf.expand_dims(input_sent_mask, axis = 2)\n",
        "sent_output = tf.multiply(input_sent_mask, sequence_output)\n",
        "sent_output = tf.keras.layers.Masking(mask_value = 0)(sent_output)\n",
        "sent_output = tf.keras.layers.LSTM(32, activation = 'relu')(sent_output)\n",
        "\n",
        "input_desc_mask = tf.expand_dims(input_desc_mask, axis = 2)\n",
        "desc_output = tf.multiply(input_desc_mask, sequence_output)\n",
        "desc_output = tf.keras.layers.Masking(mask_value = 0)(desc_output)\n",
        "desc_output = tf.keras.layers.LSTM(32, activation = 'relu')(desc_output)\n",
        "\n",
        "pooled_output = tf.keras.layers.Dense(32, activation = 'relu')(pooled_output)\n",
        "\n",
        "concat = tf.keras.layers.Concatenate(name = \"concatenate\")([sent_output, desc_output, pooled_output])\n",
        "concat = tf.keras.layers.Dropout(0.3)(concat)\n",
        "output = tf.keras.layers.Dense(1, activation = 'sigmoid', name = \"output\")(concat)\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "# Encoder-Decoder Model ---------------------\n",
        "\n",
        "model = tf.keras.models.Model(\n",
        "      inputs = [\n",
        "        input_word_ids,\n",
        "        input_mask,\n",
        "        input_type_ids,\n",
        "        input_sent_mask,\n",
        "        input_desc_mask], \n",
        "      outputs = output)\n",
        "\n",
        "model.compile(optimizer=Adam(learning_rate = 0.00005), loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GCEGsTN7lXpT"
      },
      "outputs": [],
      "source": [
        "def find_optimal_threshold(target, predicted):\n",
        "    fpr, tpr, thresholds = metrics.roc_curve(target, predicted)\n",
        "    fpr = fpr[1:]\n",
        "    tpr = tpr[1:]\n",
        "    thresholds = thresholds[1:]\n",
        "    x = tpr - fpr\n",
        "    idx = np.argmax(x)\n",
        "    return thresholds[idx]\n",
        "\n",
        "class roc_auc_threshold(tf.keras.callbacks.Callback):\n",
        "\n",
        "    def __init__(self, training_data, validation_data):\n",
        "        self.x = training_data[0]\n",
        "        self.y = training_data[1]\n",
        "        self.x_val = validation_data[0]\n",
        "        self.y_val = validation_data[1]\n",
        "\n",
        "    def on_train_begin(self, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_train_end(self, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_epoch_begin(self, epoch, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs={}):\n",
        "        model.save('model.' + str(epoch) + '.h5')\n",
        "        y_pred_val = self.model.predict(self.x_val)\n",
        "        optimal_threshold = find_optimal_threshold(self.y_val, y_pred_val)\n",
        "        y_pred_val = y_pred_val.flatten()\n",
        "        y_truth_val = self.y_val.flatten()\n",
        "        val_correct = 0\n",
        "        val_total = len(y_pred_val)\n",
        "        for i in range(len(y_pred_val)):\n",
        "          pred = 0\n",
        "          if y_pred_val[i] >= optimal_threshold:\n",
        "            pred = 1\n",
        "          if pred == y_truth_val[i]:\n",
        "            val_correct += 1\n",
        "        print(' optimal_threshold:', round(optimal_threshold, 5), '-',  \n",
        "              'val_accuracy_optimal:', round(val_correct / val_total, 5))\n",
        "        return\n",
        "\n",
        "    def on_batch_begin(self, batch, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_batch_end(self, batch, logs={}):\n",
        "        return\n",
        "\n",
        "cb = roc_auc_threshold(training_data = ([X['input_word_ids'], X['input_mask'], X['input_type_ids'], X['mask1'], X['mask2']], Y),\n",
        "                  validation_data = ([Xval['input_word_ids'], Xval['input_mask'], Xval['input_type_ids'], Xval['mask1'], Xval['mask2']], Yval))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ySbknbfzYmVf",
        "outputId": "75daa049-4995-4444-de0e-a6b89e35bfb5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/4\n",
            "134/134 [==============================] - ETA: 0s - loss: 0.6410 - accuracy: 0.6472 optimal_threshold: 0.6857 - val_accuracy_optimal: 0.73522\n",
            "134/134 [==============================] - 294s 2s/step - loss: 0.6410 - accuracy: 0.6472 - val_loss: 0.6015 - val_accuracy: 0.6787\n",
            "Epoch 2/4\n",
            "134/134 [==============================] - ETA: 0s - loss: 0.4423 - accuracy: 0.8119 optimal_threshold: 0.58551 - val_accuracy_optimal: 0.72494\n",
            "134/134 [==============================] - 274s 2s/step - loss: 0.4423 - accuracy: 0.8119 - val_loss: 0.5994 - val_accuracy: 0.7069\n",
            "Epoch 3/4\n",
            "134/134 [==============================] - ETA: 0s - loss: 0.2053 - accuracy: 0.9317 optimal_threshold: 0.59413 - val_accuracy_optimal: 0.71979\n",
            "134/134 [==============================] - 269s 2s/step - loss: 0.2053 - accuracy: 0.9317 - val_loss: 0.8349 - val_accuracy: 0.7147\n",
            "Epoch 4/4\n",
            "134/134 [==============================] - ETA: 0s - loss: 0.1355 - accuracy: 0.9513 optimal_threshold: 0.75353 - val_accuracy_optimal: 0.70951\n",
            "134/134 [==============================] - 269s 2s/step - loss: 0.1355 - accuracy: 0.9513 - val_loss: 1.0443 - val_accuracy: 0.6967\n"
          ]
        }
      ],
      "source": [
        "history = model.fit([X['input_word_ids'], X['input_mask'], X['input_type_ids'], X['mask1'], X['mask2']], Y, \n",
        "          epochs = 4, batch_size = 16, callbacks=[cb], verbose = 1,\n",
        "          validation_data = ([Xval['input_word_ids'], Xval['input_mask'], Xval['input_type_ids'], Xval['mask1'], Xval['mask2']], Yval))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ckgwg9lXO2X4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f2567c8a-fd05-4774-818c-4acbd14a1704"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        }
      ],
      "source": [
        "epoch = 0\n",
        "pretrained_model = tf.keras.models.load_model('model.' + str(epoch) + '.h5', custom_objects={'KerasLayer': hub.KerasLayer})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rA6k-oGg66nU"
      },
      "outputs": [],
      "source": [
        "Xtest, Ytest = extract_data('processed_data/test.csv', True)\n",
        "y_predicted = pretrained_model.predict([Xtest['input_word_ids'], Xtest['input_mask'], Xtest['input_type_ids'], Xtest['mask1'], Xtest['mask2']])\n",
        "y_predicted = y_predicted.flatten()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimal_threshold = 0.6857\n",
        "\n",
        "file = open('all_output.txt', 'w')\n",
        "for val in y_predicted:\n",
        "  if val < optimal_threshold:\n",
        "    print('F',file=file)\n",
        "  else:\n",
        "    print('T',file=file)\n",
        "file.close()"
      ],
      "metadata": {
        "id": "4cVE5umM-mAg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "anv-FXthURH9"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "BERT + LSTM Decoder + WE+ ROC.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8aee578f0b8d4936b2f85d81b928acd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6d032aa9257a487fafe22b164326f393",
              "IPY_MODEL_3b1b01acf06a4f1aa85f266a742263ab",
              "IPY_MODEL_9ef5c0713cc341329ad0b9c08340215c"
            ],
            "layout": "IPY_MODEL_8209ede274ec45308e6ab6d6c4ae1b05"
          }
        },
        "6d032aa9257a487fafe22b164326f393": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8190d772889240f587bbb38bdcfd98ab",
            "placeholder": "​",
            "style": "IPY_MODEL_87fdc6a90e3141dc8a257203d3c5fc58",
            "value": "Downloading: 100%"
          }
        },
        "3b1b01acf06a4f1aa85f266a742263ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ab99601e5df34514a2af1c5433bb4e17",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4ac65c9de5fa4861841c6b38a8a3afd7",
            "value": 231508
          }
        },
        "9ef5c0713cc341329ad0b9c08340215c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82745ea90a2a458c89d2dff33c324f1f",
            "placeholder": "​",
            "style": "IPY_MODEL_03ee137cd12c463fbf4c9b51807da661",
            "value": " 226k/226k [00:00&lt;00:00, 1.05MB/s]"
          }
        },
        "8209ede274ec45308e6ab6d6c4ae1b05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8190d772889240f587bbb38bdcfd98ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "87fdc6a90e3141dc8a257203d3c5fc58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ab99601e5df34514a2af1c5433bb4e17": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ac65c9de5fa4861841c6b38a8a3afd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "82745ea90a2a458c89d2dff33c324f1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "03ee137cd12c463fbf4c9b51807da661": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "48e627da295d48a499ea5f909a6074d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d7c0b325456c46ebaf75890d0d79083a",
              "IPY_MODEL_23ec4e0d866f4777a38cb2ade39f166e",
              "IPY_MODEL_7f4c5a0e3ba74acfb5c463faf1f9289a"
            ],
            "layout": "IPY_MODEL_f9cbb24742534149ac161af87a3ea0c4"
          }
        },
        "d7c0b325456c46ebaf75890d0d79083a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c1e798f6c7346bb8f56fd1894c5a618",
            "placeholder": "​",
            "style": "IPY_MODEL_066892a85d2243e4bed654ef3313c080",
            "value": "Downloading: 100%"
          }
        },
        "23ec4e0d866f4777a38cb2ade39f166e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7634b4709e5f49c9a5913e5c54dda1b4",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c750ebbdc25245f88e61c19a2d767a2b",
            "value": 28
          }
        },
        "7f4c5a0e3ba74acfb5c463faf1f9289a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71e2c4404383414d8813208d6001a02c",
            "placeholder": "​",
            "style": "IPY_MODEL_c327504bfb11482da748d6758c442e9a",
            "value": " 28.0/28.0 [00:00&lt;00:00, 706B/s]"
          }
        },
        "f9cbb24742534149ac161af87a3ea0c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c1e798f6c7346bb8f56fd1894c5a618": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "066892a85d2243e4bed654ef3313c080": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7634b4709e5f49c9a5913e5c54dda1b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c750ebbdc25245f88e61c19a2d767a2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "71e2c4404383414d8813208d6001a02c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c327504bfb11482da748d6758c442e9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7711362bff1f4efaae10e5edacf0c84c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4242ea68fa814bb59ec0f34a739e013d",
              "IPY_MODEL_0efc6092f0cf42bb87f25f0990e921c7",
              "IPY_MODEL_7ef871487a7747f18120209728034fd6"
            ],
            "layout": "IPY_MODEL_6eb2fe1e0eb94c54ba0ffd1834edba0b"
          }
        },
        "4242ea68fa814bb59ec0f34a739e013d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7b6c00dcaf644594b4ea5d21037adef1",
            "placeholder": "​",
            "style": "IPY_MODEL_72ef53eb3f5c499d927b137aa06e2620",
            "value": "Downloading: 100%"
          }
        },
        "0efc6092f0cf42bb87f25f0990e921c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e2d4a5080c64503bee3f7966cba3ddb",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b916ceb0eff24a2893d1090a5f2dfc0c",
            "value": 570
          }
        },
        "7ef871487a7747f18120209728034fd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21c5255a8c564f73ae45d620f5284867",
            "placeholder": "​",
            "style": "IPY_MODEL_1b64b483873842d3bd170b1640e5a5ba",
            "value": " 570/570 [00:00&lt;00:00, 10.8kB/s]"
          }
        },
        "6eb2fe1e0eb94c54ba0ffd1834edba0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b6c00dcaf644594b4ea5d21037adef1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72ef53eb3f5c499d927b137aa06e2620": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3e2d4a5080c64503bee3f7966cba3ddb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b916ceb0eff24a2893d1090a5f2dfc0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "21c5255a8c564f73ae45d620f5284867": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b64b483873842d3bd170b1640e5a5ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}