01_text_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision.all import *\n",
    "from fastai.learner import *\n",
    "from fastai.data.all import *\n",
    "from fastai.callback.tracker import SaveModelCallback\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from pathlib2 import Path\n",
    "import numpy as np\n",
    "import random\n",
    "from torch.nn import MSELoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can download the ebooks [here](https://www.openslr.org/resources/12/original-books.tar.gz).\n",
    "\n",
    "We will need the pretrained embeddings from https://github.com/iamyuanchung/speech2vec-pretrained-vectors. We will use them to compare our results and also to figure out what vocab the authors of the speech2vec paper used for training. Let us start with the latter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/deprecation.py:143: FutureWarning: The sklearn.datasets.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.datasets. Anything that cannot be imported from sklearn.datasets is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "from six import iteritems\n",
    "from web.datasets.similarity import fetch_MEN, fetch_WS353, fetch_SimLex999\n",
    "from web.embeddings import fetch_GloVe\n",
    "from web.evaluate import evaluate_similarity\n",
    "from web.embedding import Embedding, Vocabulary\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "speech2vec = KeyedVectors.load_word2vec_format('../speech2vec-pretrained-vectors/speech2vec/50.vec', binary=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37622"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = set(speech2vec.vocab.keys())\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "starting_lines = {\n",
    "    '1004/1004.txt.utf-8': 535,\n",
    "    '10123/10123.txt.utf-8': 85,\n",
    "    '10359/10359.txt.utf-8': 76,\n",
    "    '10360/10360.txt.utf-8': 57,\n",
    "    '10378/10378.txt.utf-8': 96,\n",
    "    '10390/10390.txt.utf-8': 89,\n",
    "    '1193/1193.txt.utf-8': 272,\n",
    "    '12441/12441-0.txt': 101,\n",
    "    '1249/1249.txt.utf-8': 614,\n",
    "    '1325/1325.txt.utf-8': 434,\n",
    "    '1674/1674.txt.utf-8': 360,\n",
    "    '2046/2046.txt.utf-8': 295,\n",
    "    '2147/2147.txt.utf-8': 66,\n",
    "    '2184/2184.txt.utf-8': 302,\n",
    "    '2383/2383.txt.utf-8': 408,\n",
    "    '2486/2486.txt.utf-8': 293,\n",
    "    '2488/2488.txt.utf-8': 495,\n",
    "    '2512/2512-0.txt': 113,\n",
    "    '2515/2515.txt.utf-8': 281,\n",
    "    '2678/2678.txt.utf-8': 305,\n",
    "    '2679/2679.txt.utf-8': 308,\n",
    "    '269/269-0.txt': 67,\n",
    "    '282/282-0.txt': 43,\n",
    "    '2891/2891.txt.utf-8': 336,\n",
    "    '3053/3053.txt.utf-8': 340,\n",
    "    '3169/3169.txt.utf-8': 386,\n",
    "    '325/325.txt.utf-8': 170,\n",
    "    '3300/3300.txt.utf-8': 40,\n",
    "    '34757/34757-0.txt': 185,\n",
    "    '3604/3604.txt.utf-8': 612,\n",
    "    '3623/3623.txt.utf-8': 342,\n",
    "    '3697/3697.txt.utf-8': 449,\n",
    "    '37660/37660-0.txt': 74,\n",
    "    '4028/4028.txt.utf-8': 402,\n",
    "    '4042/4042.txt.utf-8': 438,\n",
    "    '435/435.txt.utf-8': 62,\n",
    "    '6456/6456.txt.utf-8': 53,\n",
    "    '7098/7098.txt.utf-8': 77,\n",
    "    '76/76.txt.utf-8': 579,\n",
    "    '778/778.txt.utf-8': 297,\n",
    "    '786/786-0.txt': 158,\n",
    "}   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.8 s, sys: 520 ms, total: 13.3 s\n",
      "Wall time: 13.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "word_pairs = []\n",
    "\n",
    "for fn, starting_line in starting_lines.items():\n",
    "    with open(f'data/books/LibriSpeech/books/utf-8/{fn}') as file:\n",
    "        out = []\n",
    "        for i, line in enumerate(file.readlines()):\n",
    "            if i < starting_line: continue\n",
    "            line = line.strip()\n",
    "            toks = []\n",
    "\n",
    "            for tok in line.split():\n",
    "                tok = tok.lower()\n",
    "                if tok in vocab:\n",
    "                    toks.append(tok)\n",
    "                else:\n",
    "                    toks.append('<UNK>')\n",
    "            out += toks\n",
    "\n",
    "        for i, word in enumerate(out):\n",
    "            if word is '<UNK>': continue\n",
    "            for offset in [-2, -1, 1, 2]:\n",
    "                if i + offset < 0 or i + offset >= len(out): continue\n",
    "                target_word = out[i+offset]\n",
    "                if target_word is '<UNK>': continue\n",
    "                word_pairs.append([word, target_word])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['midway', 'upon'],\n",
       " ['midway', 'the'],\n",
       " ['upon', 'midway'],\n",
       " ['upon', 'the'],\n",
       " ['upon', 'journey'],\n",
       " ['the', 'midway'],\n",
       " ['the', 'upon'],\n",
       " ['the', 'journey'],\n",
       " ['the', 'of'],\n",
       " ['journey', 'upon'],\n",
       " ['journey', 'the'],\n",
       " ['journey', 'of'],\n",
       " ['journey', 'our'],\n",
       " ['of', 'the'],\n",
       " ['of', 'journey'],\n",
       " ['of', 'our'],\n",
       " ['of', 'life'],\n",
       " ['our', 'journey'],\n",
       " ['our', 'of'],\n",
       " ['our', 'life']]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_pairs[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10140726"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = list(vocab)\n",
    "np.random.shuffle(vocab)\n",
    "word2index = {w: i for i, w in enumerate(vocab)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset():\n",
    "    def __init__(self, pairs):\n",
    "        self.pairs = pairs\n",
    "    def __len__(self):\n",
    "        return len(self.pairs)\n",
    "    def __getitem__(self, idx):\n",
    "        source_word, target_word = self.pairs[idx]\n",
    "        return word2index[source_word], word2index[target_word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = Dataset(word_pairs[:9_500_000])\n",
    "valid_ds = Dataset(word_pairs[9_500_000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9500000, 640726)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ds), len(valid_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "BS = 2048\n",
    "NUM_WORKERS = 8\n",
    "\n",
    "train_dl = DataLoader(train_ds, BS, NUM_WORKERS, shuffle=True)\n",
    "valid_dl = DataLoader(valid_ds, BS, NUM_WORKERS)\n",
    "\n",
    "dls = DataLoaders(train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(Module):\n",
    "    def __init__(self, hidden_size=50):\n",
    "        self.embeddings =nn.Embedding(len(vocab), hidden_size)\n",
    "        self.linear = nn.Linear(hidden_size, len(vocab))\n",
    "            \n",
    "    def forward(self, x):\n",
    "        x = self.embeddings(x)\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(\n",
    "    dls.cuda(),\n",
    "    Model().cuda(),\n",
    "    loss_func=CrossEntropyLossFlat(),\n",
    "    opt_func=Adam,\n",
    "    metrics=[accuracy]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='116' class='' max='120' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      96.67% [116/120 2:08:39<04:26]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>6.398363</td>\n",
       "      <td>6.434661</td>\n",
       "      <td>0.069900</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>6.314659</td>\n",
       "      <td>6.351384</td>\n",
       "      <td>0.073075</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>6.268547</td>\n",
       "      <td>6.318035</td>\n",
       "      <td>0.074938</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.234494</td>\n",
       "      <td>6.297647</td>\n",
       "      <td>0.075747</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.205285</td>\n",
       "      <td>6.281332</td>\n",
       "      <td>0.076509</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.213039</td>\n",
       "      <td>6.271708</td>\n",
       "      <td>0.076711</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6.190912</td>\n",
       "      <td>6.262813</td>\n",
       "      <td>0.078213</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>6.181589</td>\n",
       "      <td>6.255394</td>\n",
       "      <td>0.079232</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>6.171001</td>\n",
       "      <td>6.247848</td>\n",
       "      <td>0.077217</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>6.163467</td>\n",
       "      <td>6.242204</td>\n",
       "      <td>0.079541</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>6.163079</td>\n",
       "      <td>6.235297</td>\n",
       "      <td>0.080104</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>6.146518</td>\n",
       "      <td>6.232151</td>\n",
       "      <td>0.079199</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>6.146552</td>\n",
       "      <td>6.229044</td>\n",
       "      <td>0.079619</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>6.144467</td>\n",
       "      <td>6.226439</td>\n",
       "      <td>0.080087</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>6.132597</td>\n",
       "      <td>6.222215</td>\n",
       "      <td>0.080164</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>6.126264</td>\n",
       "      <td>6.218241</td>\n",
       "      <td>0.080098</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>6.130122</td>\n",
       "      <td>6.216129</td>\n",
       "      <td>0.079135</td>\n",
       "      <td>01:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>6.115284</td>\n",
       "      <td>6.213651</td>\n",
       "      <td>0.080178</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>6.125466</td>\n",
       "      <td>6.211181</td>\n",
       "      <td>0.080335</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>6.098159</td>\n",
       "      <td>6.209764</td>\n",
       "      <td>0.079969</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>6.121174</td>\n",
       "      <td>6.209696</td>\n",
       "      <td>0.080164</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>6.114490</td>\n",
       "      <td>6.204386</td>\n",
       "      <td>0.081614</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>6.105857</td>\n",
       "      <td>6.205492</td>\n",
       "      <td>0.080638</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>6.100382</td>\n",
       "      <td>6.204086</td>\n",
       "      <td>0.081138</td>\n",
       "      <td>01:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>6.096159</td>\n",
       "      <td>6.201629</td>\n",
       "      <td>0.081885</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>6.089718</td>\n",
       "      <td>6.201606</td>\n",
       "      <td>0.080604</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>6.083433</td>\n",
       "      <td>6.199481</td>\n",
       "      <td>0.080838</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>6.090341</td>\n",
       "      <td>6.200011</td>\n",
       "      <td>0.081704</td>\n",
       "      <td>01:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>6.081747</td>\n",
       "      <td>6.198062</td>\n",
       "      <td>0.081108</td>\n",
       "      <td>01:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>6.083040</td>\n",
       "      <td>6.199063</td>\n",
       "      <td>0.080712</td>\n",
       "      <td>01:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>6.084527</td>\n",
       "      <td>6.198118</td>\n",
       "      <td>0.081885</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>6.093029</td>\n",
       "      <td>6.196053</td>\n",
       "      <td>0.081684</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>6.077403</td>\n",
       "      <td>6.198280</td>\n",
       "      <td>0.080854</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>6.084495</td>\n",
       "      <td>6.197473</td>\n",
       "      <td>0.080154</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>6.087610</td>\n",
       "      <td>6.196152</td>\n",
       "      <td>0.080791</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>6.078666</td>\n",
       "      <td>6.197339</td>\n",
       "      <td>0.079558</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>6.077626</td>\n",
       "      <td>6.197272</td>\n",
       "      <td>0.081111</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>6.075151</td>\n",
       "      <td>6.195645</td>\n",
       "      <td>0.081089</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>6.070219</td>\n",
       "      <td>6.196524</td>\n",
       "      <td>0.081088</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>6.069575</td>\n",
       "      <td>6.195593</td>\n",
       "      <td>0.081701</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>6.064618</td>\n",
       "      <td>6.196311</td>\n",
       "      <td>0.080602</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>6.067647</td>\n",
       "      <td>6.196920</td>\n",
       "      <td>0.080989</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>6.065550</td>\n",
       "      <td>6.196130</td>\n",
       "      <td>0.079714</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>6.080386</td>\n",
       "      <td>6.196251</td>\n",
       "      <td>0.081158</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>6.062389</td>\n",
       "      <td>6.196629</td>\n",
       "      <td>0.081776</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>6.068981</td>\n",
       "      <td>6.195887</td>\n",
       "      <td>0.081512</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>6.082456</td>\n",
       "      <td>6.196513</td>\n",
       "      <td>0.081732</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>6.072451</td>\n",
       "      <td>6.195545</td>\n",
       "      <td>0.080927</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>6.065154</td>\n",
       "      <td>6.195201</td>\n",
       "      <td>0.081287</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>6.059655</td>\n",
       "      <td>6.196261</td>\n",
       "      <td>0.081826</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>6.070344</td>\n",
       "      <td>6.197176</td>\n",
       "      <td>0.080560</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>6.058672</td>\n",
       "      <td>6.197319</td>\n",
       "      <td>0.080958</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>6.067169</td>\n",
       "      <td>6.196756</td>\n",
       "      <td>0.080996</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>6.066325</td>\n",
       "      <td>6.195302</td>\n",
       "      <td>0.080805</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>6.067095</td>\n",
       "      <td>6.197198</td>\n",
       "      <td>0.081061</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>6.062634</td>\n",
       "      <td>6.195850</td>\n",
       "      <td>0.081454</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>6.050324</td>\n",
       "      <td>6.196020</td>\n",
       "      <td>0.081390</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>6.045051</td>\n",
       "      <td>6.195857</td>\n",
       "      <td>0.080488</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>6.065918</td>\n",
       "      <td>6.196918</td>\n",
       "      <td>0.080641</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>6.056967</td>\n",
       "      <td>6.196938</td>\n",
       "      <td>0.080732</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>6.048471</td>\n",
       "      <td>6.196724</td>\n",
       "      <td>0.079970</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61</td>\n",
       "      <td>6.058390</td>\n",
       "      <td>6.196484</td>\n",
       "      <td>0.080935</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62</td>\n",
       "      <td>6.052114</td>\n",
       "      <td>6.196850</td>\n",
       "      <td>0.081273</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63</td>\n",
       "      <td>6.054093</td>\n",
       "      <td>6.196129</td>\n",
       "      <td>0.081163</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64</td>\n",
       "      <td>6.053756</td>\n",
       "      <td>6.196205</td>\n",
       "      <td>0.080663</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65</td>\n",
       "      <td>6.063205</td>\n",
       "      <td>6.195754</td>\n",
       "      <td>0.081014</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66</td>\n",
       "      <td>6.064188</td>\n",
       "      <td>6.197370</td>\n",
       "      <td>0.082107</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67</td>\n",
       "      <td>6.057389</td>\n",
       "      <td>6.196077</td>\n",
       "      <td>0.081620</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68</td>\n",
       "      <td>6.059843</td>\n",
       "      <td>6.196392</td>\n",
       "      <td>0.081756</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69</td>\n",
       "      <td>6.060343</td>\n",
       "      <td>6.196908</td>\n",
       "      <td>0.080944</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>6.064067</td>\n",
       "      <td>6.197651</td>\n",
       "      <td>0.080359</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>6.056321</td>\n",
       "      <td>6.196720</td>\n",
       "      <td>0.082486</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>6.067625</td>\n",
       "      <td>6.196831</td>\n",
       "      <td>0.081389</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>6.071109</td>\n",
       "      <td>6.196712</td>\n",
       "      <td>0.081559</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>6.055758</td>\n",
       "      <td>6.196496</td>\n",
       "      <td>0.080599</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>6.051794</td>\n",
       "      <td>6.196199</td>\n",
       "      <td>0.081895</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76</td>\n",
       "      <td>6.044382</td>\n",
       "      <td>6.196485</td>\n",
       "      <td>0.082302</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77</td>\n",
       "      <td>6.054579</td>\n",
       "      <td>6.197041</td>\n",
       "      <td>0.081083</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78</td>\n",
       "      <td>6.055796</td>\n",
       "      <td>6.197724</td>\n",
       "      <td>0.080168</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>79</td>\n",
       "      <td>6.046536</td>\n",
       "      <td>6.198373</td>\n",
       "      <td>0.081490</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>6.066472</td>\n",
       "      <td>6.197482</td>\n",
       "      <td>0.081345</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81</td>\n",
       "      <td>6.060904</td>\n",
       "      <td>6.197263</td>\n",
       "      <td>0.080637</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>82</td>\n",
       "      <td>6.043007</td>\n",
       "      <td>6.197736</td>\n",
       "      <td>0.079708</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>83</td>\n",
       "      <td>6.049100</td>\n",
       "      <td>6.197619</td>\n",
       "      <td>0.080685</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>84</td>\n",
       "      <td>6.057712</td>\n",
       "      <td>6.196965</td>\n",
       "      <td>0.081286</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85</td>\n",
       "      <td>6.047692</td>\n",
       "      <td>6.196792</td>\n",
       "      <td>0.080977</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>86</td>\n",
       "      <td>6.058916</td>\n",
       "      <td>6.197378</td>\n",
       "      <td>0.082255</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>87</td>\n",
       "      <td>6.052837</td>\n",
       "      <td>6.197021</td>\n",
       "      <td>0.081353</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>88</td>\n",
       "      <td>6.054870</td>\n",
       "      <td>6.197420</td>\n",
       "      <td>0.081149</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>89</td>\n",
       "      <td>6.059271</td>\n",
       "      <td>6.196539</td>\n",
       "      <td>0.081376</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>6.053653</td>\n",
       "      <td>6.198144</td>\n",
       "      <td>0.081934</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>91</td>\n",
       "      <td>6.048151</td>\n",
       "      <td>6.197707</td>\n",
       "      <td>0.080543</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>92</td>\n",
       "      <td>6.044680</td>\n",
       "      <td>6.197469</td>\n",
       "      <td>0.081147</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>93</td>\n",
       "      <td>6.052677</td>\n",
       "      <td>6.198841</td>\n",
       "      <td>0.081287</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>94</td>\n",
       "      <td>6.046137</td>\n",
       "      <td>6.197628</td>\n",
       "      <td>0.080946</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95</td>\n",
       "      <td>6.064752</td>\n",
       "      <td>6.197173</td>\n",
       "      <td>0.080997</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>6.063368</td>\n",
       "      <td>6.197433</td>\n",
       "      <td>0.081453</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>97</td>\n",
       "      <td>6.051879</td>\n",
       "      <td>6.197284</td>\n",
       "      <td>0.081520</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>6.052802</td>\n",
       "      <td>6.197692</td>\n",
       "      <td>0.080866</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>6.053977</td>\n",
       "      <td>6.198285</td>\n",
       "      <td>0.081442</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>6.049850</td>\n",
       "      <td>6.198081</td>\n",
       "      <td>0.080591</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>101</td>\n",
       "      <td>6.049696</td>\n",
       "      <td>6.197280</td>\n",
       "      <td>0.080886</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>102</td>\n",
       "      <td>6.060956</td>\n",
       "      <td>6.196677</td>\n",
       "      <td>0.080865</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>103</td>\n",
       "      <td>6.055011</td>\n",
       "      <td>6.197059</td>\n",
       "      <td>0.082149</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>104</td>\n",
       "      <td>6.054603</td>\n",
       "      <td>6.197885</td>\n",
       "      <td>0.080754</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>105</td>\n",
       "      <td>6.042034</td>\n",
       "      <td>6.198215</td>\n",
       "      <td>0.081643</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>106</td>\n",
       "      <td>6.046163</td>\n",
       "      <td>6.197485</td>\n",
       "      <td>0.081582</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>107</td>\n",
       "      <td>6.051970</td>\n",
       "      <td>6.198274</td>\n",
       "      <td>0.081058</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>108</td>\n",
       "      <td>6.052937</td>\n",
       "      <td>6.198036</td>\n",
       "      <td>0.082082</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>109</td>\n",
       "      <td>6.044799</td>\n",
       "      <td>6.197468</td>\n",
       "      <td>0.081845</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>6.044699</td>\n",
       "      <td>6.198159</td>\n",
       "      <td>0.082307</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>111</td>\n",
       "      <td>6.050252</td>\n",
       "      <td>6.197940</td>\n",
       "      <td>0.080772</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>112</td>\n",
       "      <td>6.038146</td>\n",
       "      <td>6.198319</td>\n",
       "      <td>0.080476</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>113</td>\n",
       "      <td>6.052275</td>\n",
       "      <td>6.198242</td>\n",
       "      <td>0.081554</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>114</td>\n",
       "      <td>6.044151</td>\n",
       "      <td>6.197045</td>\n",
       "      <td>0.081838</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>115</td>\n",
       "      <td>6.051582</td>\n",
       "      <td>6.198220</td>\n",
       "      <td>0.081269</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='406' class='' max='4639' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      8.75% [406/4639 00:17<03:00 6.0272]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "NUM_EPOCHS = 120\n",
    "learn.fit(NUM_EPOCHS, lr=1e-3, cbs=SaveModelCallback(fname='text_embeddings', every_epoch=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evalute embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = learn.model.embeddings.weight.cpu().detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "e = Embeddings(\n",
    "    embeddings,\n",
    "    vocab\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fast: ['fast', 'forth', 'lightly', 'docile', 'softly']\n",
      "lost: ['lost', 'won', 'misjudged', 'caught', 'changed']\n",
      "small: ['small', 'large', 'great', 'packet', 'solid']\n",
      "true: ['true', 'perfect', 'glorious', 'essence', 'sooth']\n",
      "crazy: ['crazy', 'mad', 'hucks', 'handy', 'boswell']\n",
      "slow: ['slow', 'rapid', 'civilisation', 'stormy', 'impetuous']\n"
     ]
    }
   ],
   "source": [
    "for w in ['fast', 'lost', 'small', 'true', 'crazy', 'slow']:\n",
    "    print(f'{w}: {e.nn_words_to(e[w])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating embeddings using [word-embeddings-benchmarks](https://github.com/kudkudak/word-embeddings-benchmarks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from six import iteritems\n",
    "from web.datasets.similarity import fetch_MEN, fetch_WS353, fetch_SimLex999\n",
    "from web.embeddings import fetch_GloVe\n",
    "from web.evaluate import evaluate_similarity\n",
    "from web.embedding import Embedding, Vocabulary\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = {\n",
    "    \"MEN\": fetch_MEN(),\n",
    "    \"WS353\": fetch_WS353(),\n",
    "    \"SIMLEX999\": fetch_SimLex999()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "our_embeddings = Embedding(\n",
    "    Vocabulary(vocab),\n",
    "    embeddings\n",
    ")\n",
    "\n",
    "speech2vec = KeyedVectors.load_word2vec_format('../speech2vec-pretrained-vectors/word2vec/50.vec', binary=False) \n",
    "speech2vec_embeddings = Embedding(Vocabulary(list(speech2vec.vocab.keys())), speech2vec.vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Missing 392 words. Will replace them with mean vector\n",
      "/opt/conda/lib/python3.7/site-packages/web-0.0.1-py3.7.egg/web/evaluate.py:336: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  A = np.vstack(w.get(word, mean_vector) for word in X[:, 0])\n",
      "/opt/conda/lib/python3.7/site-packages/web-0.0.1-py3.7.egg/web/evaluate.py:337: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  B = np.vstack(w.get(word, mean_vector) for word in X[:, 1])\n",
      "Missing 61 words. Will replace them with mean vector\n",
      "Missing 24 words. Will replace them with mean vector\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spearman correlation of scores on MEN 0.32009252155468704\n",
      "Spearman correlation of scores on WS353 0.28296014417794485\n",
      "Spearman correlation of scores on SIMLEX999 0.12822289208904286\n"
     ]
    }
   ],
   "source": [
    "for name, data in iteritems(tasks):\n",
    "    print(\"Spearman correlation of scores on {} {}\".format(name, evaluate_similarity(our_embeddings, data.X, data.y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Missing 392 words. Will replace them with mean vector\n",
      "/opt/conda/lib/python3.7/site-packages/web-0.0.1-py3.7.egg/web/evaluate.py:336: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  A = np.vstack(w.get(word, mean_vector) for word in X[:, 0])\n",
      "/opt/conda/lib/python3.7/site-packages/web-0.0.1-py3.7.egg/web/evaluate.py:337: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  B = np.vstack(w.get(word, mean_vector) for word in X[:, 1])\n",
      "Missing 61 words. Will replace them with mean vector\n",
      "Missing 24 words. Will replace them with mean vector\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spearman correlation of scores on MEN 0.6056592803599269\n",
      "Spearman correlation of scores on WS353 0.43349390636024643\n",
      "Spearman correlation of scores on SIMLEX999 0.25938770901422736\n"
     ]
    }
   ],
   "source": [
    "for name, data in iteritems(tasks):\n",
    "    print(\"Spearman correlation of scores on {} {}\".format(name, evaluate_similarity(speech2vec_embeddings, data.X, data.y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss decrease and improvements on semantic tasks as training progresses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "val_losses, accuracies, task_perf = [], [], []\n",
    "for i in range(NUM_EPOCHS):\n",
    "    learn.load(f'text_embeddings_{i}')\n",
    "    loss, accuracy = learn.validate()\n",
    "    val_losses.append(loss)\n",
    "    accuracies.append(accuracy)\n",
    "    \n",
    "    embeddings = learn.model.embeddings.weight.cpu().detach().numpy()\n",
    "    our_embeddings = Embedding(\n",
    "        Vocabulary([w.lower() for w in vocab]),\n",
    "        embeddings\n",
    "    )\n",
    "\n",
    "    task_perf.append([evaluate_similarity(our_embeddings, data.X, data.y) for name, data in iteritems(tasks)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "men, ws353, simlex999 = list(zip(*task_perf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax1 = plt.subplots()\n",
    "\n",
    "ax1.plot(val_losses, label='val loss')\n",
    "\n",
    "ax2 = ax1.twinx()\n",
    "ax2.plot(men, label='MEN', c='g')\n",
    "ax2.plot(ws353, label='WS353', c='m')\n",
    "ax2.plot(simlex999, label='SIMLEX999', c='y')\n",
    "\n",
    "ax1.legend(loc=[0.07, 0.9])\n",
    "ax2.legend(loc=[0.7, 0.15])\n",
    "\n",
    "ax1.set_xlabel('epochs');"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}