01b_text_embeddings_balanced_epochs_better_vocab.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision.all import *\n",
    "from fastai.learner import *\n",
    "from fastai.data.all import *\n",
    "from fastai.callback.tracker import SaveModelCallback\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from pathlib2 import Path\n",
    "import numpy as np\n",
    "import random\n",
    "from torch.nn import MSELoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can download the ebooks [here](https://www.openslr.org/resources/12/original-books.tar.gz).\n",
    "\n",
    "We will need the pretrained embeddings from https://github.com/iamyuanchung/speech2vec-pretrained-vectors. We will use them to compare our results and also to figure out what vocab the authors of the speech2vec paper used for training. Let us start with the latter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/deprecation.py:143: FutureWarning: The sklearn.datasets.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.datasets. Anything that cannot be imported from sklearn.datasets is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "from six import iteritems\n",
    "from web.datasets.similarity import fetch_MEN, fetch_WS353, fetch_SimLex999\n",
    "from web.embeddings import fetch_GloVe\n",
    "from web.evaluate import evaluate_similarity\n",
    "from web.embedding import Embedding, Vocabulary\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = {\n",
    "    \"MEN\": fetch_MEN(),\n",
    "    \"WS353\": fetch_WS353(),\n",
    "    \"SIMLEX999\": fetch_SimLex999()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "starting_lines = {\n",
    "    '1004/1004.txt.utf-8': 535,\n",
    "    '10123/10123.txt.utf-8': 85,\n",
    "    '10359/10359.txt.utf-8': 76,\n",
    "    '10360/10360.txt.utf-8': 57,\n",
    "    '10378/10378.txt.utf-8': 96,\n",
    "    '10390/10390.txt.utf-8': 89,\n",
    "    '1193/1193.txt.utf-8': 272,\n",
    "    '12441/12441-0.txt': 101,\n",
    "    '1249/1249.txt.utf-8': 614,\n",
    "    '1325/1325.txt.utf-8': 434,\n",
    "    '1674/1674.txt.utf-8': 360,\n",
    "    '2046/2046.txt.utf-8': 295,\n",
    "    '2147/2147.txt.utf-8': 66,\n",
    "    '2184/2184.txt.utf-8': 302,\n",
    "    '2383/2383.txt.utf-8': 408,\n",
    "    '2486/2486.txt.utf-8': 293,\n",
    "    '2488/2488.txt.utf-8': 495,\n",
    "    '2512/2512-0.txt': 113,\n",
    "    '2515/2515.txt.utf-8': 281,\n",
    "    '2678/2678.txt.utf-8': 305,\n",
    "    '2679/2679.txt.utf-8': 308,\n",
    "    '269/269-0.txt': 67,\n",
    "    '282/282-0.txt': 43,\n",
    "    '2891/2891.txt.utf-8': 336,\n",
    "    '3053/3053.txt.utf-8': 340,\n",
    "    '3169/3169.txt.utf-8': 386,\n",
    "    '325/325.txt.utf-8': 170,\n",
    "    '3300/3300.txt.utf-8': 40,\n",
    "    '34757/34757-0.txt': 185,\n",
    "    '3604/3604.txt.utf-8': 612,\n",
    "    '3623/3623.txt.utf-8': 342,\n",
    "    '3697/3697.txt.utf-8': 449,\n",
    "    '37660/37660-0.txt': 74,\n",
    "    '4028/4028.txt.utf-8': 402,\n",
    "    '4042/4042.txt.utf-8': 438,\n",
    "    '435/435.txt.utf-8': 62,\n",
    "    '6456/6456.txt.utf-8': 53,\n",
    "    '7098/7098.txt.utf-8': 77,\n",
    "    '76/76.txt.utf-8': 579,\n",
    "    '778/778.txt.utf-8': 297,\n",
    "    '786/786-0.txt': 158,\n",
    "}   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 76 ms, sys: 44 ms, total: 120 ms\n",
      "Wall time: 120 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "text = ''\n",
    "\n",
    "for fn, starting_line in starting_lines.items():\n",
    "    with open(f'data/books/LibriSpeech/books/utf-8/{fn}') as file:\n",
    "        lines = file.readlines()[starting_line:]\n",
    "        text += ' '.join(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "doc = nlp.tokenizer(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [token.text.lower() for token in doc if token.is_punct != True and str(token).isalpha()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3694374"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60364"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = Counter(words)\n",
    "len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['midway',\n",
       " 'upon',\n",
       " 'the',\n",
       " 'journey',\n",
       " 'of',\n",
       " 'our',\n",
       " 'life',\n",
       " 'i',\n",
       " 'found',\n",
       " 'myself',\n",
       " 'within',\n",
       " 'a',\n",
       " 'forest',\n",
       " 'dark',\n",
       " 'for',\n",
       " 'the',\n",
       " 'straightforward',\n",
       " 'pathway',\n",
       " 'had',\n",
       " 'been']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "speech2vec = KeyedVectors.load_word2vec_format('../speech2vec-pretrained-vectors/speech2vec/50.vec', binary=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "speech2vec_vocab = set(speech2vec.vocab.keys())\n",
    "\n",
    "# vocab = [w for w, c in c.most_common() if c > 25]\n",
    "vocab = list(set(words))\n",
    "vocab = [w for w in vocab if w in speech2vec_vocab]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29876"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3694374"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.9 s, sys: 608 ms, total: 14.5 s\n",
      "Wall time: 14.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "word_pairs = []\n",
    "offsets = [-2, -1, 1, 2]\n",
    "vocab_set = set(vocab)\n",
    "\n",
    "for i, source_w in enumerate(words):\n",
    "    if source_w not in vocab_set: continue\n",
    "    for o in offsets:\n",
    "        try:\n",
    "            target_w = words[i+o]\n",
    "        except: continue\n",
    "        if target_w in vocab_set:\n",
    "            word_pairs.append([source_w, target_w])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14129207"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.random.shuffle(word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from collections import Counter\n",
    "\n",
    "# c = Counter(all_words)\n",
    "\n",
    "# vocab = [w for w, count in c.most_common() if count >= 50]\n",
    "\n",
    "# len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from collections import Counter\n",
    "\n",
    "# c = Counter([w0 for w0, w1 in word_pairs])\n",
    "\n",
    "# missing = []\n",
    "# for w, count in c.items():\n",
    "#     if count < 25: missing.append(w)\n",
    "\n",
    "# ', '.join([f'\"{w}\"' for w in missing])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "for w1, w2 in word_pairs:\n",
    "    vocab.add(w1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = list(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2index = {w: i for i, w in enumerate(vocab)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset():\n",
    "    def __init__(self, pairs, len_mult=1000):\n",
    "        self.pairs = pairs\n",
    "        self.len_mult = len_mult\n",
    "        self.word2row_idxs = defaultdict(list)\n",
    "\n",
    "        for idx, (source_word, target_word) in enumerate(pairs):\n",
    "            self.word2row_idxs[source_word].append(idx)\n",
    "    def __len__(self):\n",
    "        return len(vocab) * self.len_mult\n",
    "    def __getitem__(self, idx):\n",
    "        word_idx = idx % len(vocab)\n",
    "        word = vocab[word_idx]\n",
    "        row_idx = np.random.choice(self.word2row_idxs[word])\n",
    "        source_word, target_word = self.pairs[row_idx]\n",
    "        return word2index[source_word], word2index[target_word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = Dataset(word_pairs)\n",
    "valid_ds = Dataset(word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "missing = []\n",
    "for w, idxs in train_ds.word2row_idxs.items():\n",
    "    if len(idxs) == 0: missing.append(w)\n",
    "for w, idxs in  valid_ds.word2row_idxs.items():\n",
    "    if len(idxs) == 0 : missing.append(w)\n",
    "\n",
    "print(', '.join([f'\"{w}\"' for w in missing]))\n",
    "\n",
    "for i in range(len(vocab)):\n",
    "    train_ds[i]\n",
    "    \n",
    "for i in range(len(vocab)):\n",
    "    valid_ds[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29873000, 29873000)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ds), len(valid_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "BS = 2048\n",
    "NUM_WORKERS = 8\n",
    "\n",
    "train_dl = DataLoader(train_ds, BS, NUM_WORKERS, shuffle=True)\n",
    "valid_dl = DataLoader(valid_ds, BS, NUM_WORKERS)\n",
    "\n",
    "dls = DataLoaders(train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(Module):\n",
    "    def __init__(self, hidden_size=50):\n",
    "        self.embeddings = nn.Embedding(len(vocab), hidden_size)\n",
    "        self.linear = nn.Linear(hidden_size, len(vocab))\n",
    "            \n",
    "    def forward(self, x):\n",
    "        x = self.embeddings(x)\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(\n",
    "    dls.cuda(),\n",
    "    Model().cuda(),\n",
    "    loss_func=CrossEntropyLossFlat(),\n",
    "    opt_func=Adam,\n",
    "    metrics=[accuracy]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.435266</td>\n",
       "      <td>5.448726</td>\n",
       "      <td>0.113236</td>\n",
       "      <td>18:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.914404</td>\n",
       "      <td>4.902357</td>\n",
       "      <td>0.132039</td>\n",
       "      <td>18:23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.581921</td>\n",
       "      <td>4.579188</td>\n",
       "      <td>0.140887</td>\n",
       "      <td>17:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.400144</td>\n",
       "      <td>4.401237</td>\n",
       "      <td>0.144777</td>\n",
       "      <td>17:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.302540</td>\n",
       "      <td>4.299611</td>\n",
       "      <td>0.146375</td>\n",
       "      <td>18:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>4.248083</td>\n",
       "      <td>4.239969</td>\n",
       "      <td>0.146958</td>\n",
       "      <td>17:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>4.203971</td>\n",
       "      <td>4.201066</td>\n",
       "      <td>0.147572</td>\n",
       "      <td>17:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4.170121</td>\n",
       "      <td>4.174359</td>\n",
       "      <td>0.147535</td>\n",
       "      <td>17:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>4.152227</td>\n",
       "      <td>4.154513</td>\n",
       "      <td>0.147813</td>\n",
       "      <td>18:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>4.137247</td>\n",
       "      <td>4.139415</td>\n",
       "      <td>0.147880</td>\n",
       "      <td>17:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>4.126031</td>\n",
       "      <td>4.127851</td>\n",
       "      <td>0.148032</td>\n",
       "      <td>17:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>4.113798</td>\n",
       "      <td>4.118011</td>\n",
       "      <td>0.147839</td>\n",
       "      <td>17:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>4.103366</td>\n",
       "      <td>4.109913</td>\n",
       "      <td>0.148082</td>\n",
       "      <td>17:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>4.108755</td>\n",
       "      <td>4.103797</td>\n",
       "      <td>0.147986</td>\n",
       "      <td>18:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>4.095116</td>\n",
       "      <td>4.098682</td>\n",
       "      <td>0.148067</td>\n",
       "      <td>18:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>4.090362</td>\n",
       "      <td>4.094503</td>\n",
       "      <td>0.148218</td>\n",
       "      <td>18:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>4.090077</td>\n",
       "      <td>4.090020</td>\n",
       "      <td>0.147954</td>\n",
       "      <td>19:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>4.078089</td>\n",
       "      <td>4.086153</td>\n",
       "      <td>0.147997</td>\n",
       "      <td>19:29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>4.091398</td>\n",
       "      <td>4.082799</td>\n",
       "      <td>0.148084</td>\n",
       "      <td>19:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>4.082317</td>\n",
       "      <td>4.079564</td>\n",
       "      <td>0.148190</td>\n",
       "      <td>18:58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "NUM_EPOCHS = 20\n",
    "learn.fit(NUM_EPOCHS, lr=1e-3, cbs=SaveModelCallback(fname='text_embeddings', every_epoch=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evalute embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = learn.model.embeddings.weight.cpu().detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "e = Embeddings(\n",
    "    embeddings,\n",
    "    vocab\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fast: ['fast', 'down', 'lay', 'looked', 'just']\n",
      "lost: ['lost', 'yet', 'that', 'utterly', 'since']\n",
      "small: ['small', 'large', 'fine', 'vast', 'travelling']\n",
      "true: ['true', 'truth', 'that', 'indeed', 'best']\n",
      "nice: ['nice', 'foolish', 'clever', 'hard', 'comfortable']\n",
      "slow: ['slow', 'swift', 'moving', 'strong', 'sound']\n"
     ]
    }
   ],
   "source": [
    "for w in ['fast', 'lost', 'small', 'true', 'nice', 'slow']:\n",
    "    print(f'{w}: {e.nn_words_to(e[w])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating embeddings using [word-embeddings-benchmarks](https://github.com/kudkudak/word-embeddings-benchmarks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from six import iteritems\n",
    "from web.datasets.similarity import fetch_MEN, fetch_WS353, fetch_SimLex999\n",
    "from web.embeddings import fetch_GloVe\n",
    "from web.evaluate import evaluate_similarity\n",
    "from web.embedding import Embedding, Vocabulary\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = {\n",
    "    \"MEN\": fetch_MEN(),\n",
    "    \"WS353\": fetch_WS353(),\n",
    "    \"SIMLEX999\": fetch_SimLex999()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "our_embeddings = Embedding(\n",
    "    Vocabulary(vocab),\n",
    "    embeddings\n",
    ")\n",
    "\n",
    "speech2vec = KeyedVectors.load_word2vec_format('../speech2vec-pretrained-vectors/word2vec/50.vec', binary=False) \n",
    "speech2vec_embeddings = Embedding(Vocabulary(list(speech2vec.vocab.keys())), speech2vec.vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Missing 530 words. Will replace them with mean vector\n",
      "/opt/conda/lib/python3.7/site-packages/web-0.0.1-py3.7.egg/web/evaluate.py:336: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  A = np.vstack(w.get(word, mean_vector) for word in X[:, 0])\n",
      "/opt/conda/lib/python3.7/site-packages/web-0.0.1-py3.7.egg/web/evaluate.py:337: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  B = np.vstack(w.get(word, mean_vector) for word in X[:, 1])\n",
      "Missing 75 words. Will replace them with mean vector\n",
      "Missing 36 words. Will replace them with mean vector\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spearman correlation of scores on MEN 0.25316331304609385\n",
      "Spearman correlation of scores on WS353 0.1194036009537742\n",
      "Spearman correlation of scores on SIMLEX999 0.04217566566804081\n"
     ]
    }
   ],
   "source": [
    "for name, data in iteritems(tasks):\n",
    "    print(\"Spearman correlation of scores on {} {}\".format(name, evaluate_similarity(our_embeddings, data.X, data.y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Missing 392 words. Will replace them with mean vector\n",
      "Missing 61 words. Will replace them with mean vector\n",
      "Missing 24 words. Will replace them with mean vector\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spearman correlation of scores on MEN 0.6056592803599269\n",
      "Spearman correlation of scores on WS353 0.43349390636024643\n",
      "Spearman correlation of scores on SIMLEX999 0.25938770901422736\n"
     ]
    }
   ],
   "source": [
    "for name, data in iteritems(tasks):\n",
    "    print(\"Spearman correlation of scores on {} {}\".format(name, evaluate_similarity(speech2vec_embeddings, data.X, data.y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss decrease and improvements on semantic tasks as training progresses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "val_losses, accuracies, task_perf = [], [], []\n",
    "for i in range(NUM_EPOCHS):\n",
    "    learn.load(f'text_embeddings_{i}')\n",
    "    loss, accuracy = learn.validate()\n",
    "    val_losses.append(loss)\n",
    "    accuracies.append(accuracy)\n",
    "    \n",
    "    embeddings = learn.model.embeddings.weight.cpu().detach().numpy()\n",
    "    our_embeddings = Embedding(\n",
    "        Vocabulary([w.lower() for w in vocab]),\n",
    "        embeddings\n",
    "    )\n",
    "\n",
    "    task_perf.append([evaluate_similarity(our_embeddings, data.X, data.y) for name, data in iteritems(tasks)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "men, ws353, simlex999 = list(zip(*task_perf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEGCAYAAAC3lehYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU1cH/8c+ZPctksu8QguyQgArIJqi4AAKKYsGFanFti0vda/21T219qvXxqVVxe6xitWhFRBEUtCgCsggqe9hJQvZ9zySznN8fMwlJyDJJJpks5/16zWvuzL1z5ySE+c5Zr5BSoiiKoihdQePrAiiKoih9lwoZRVEUpcuokFEURVG6jAoZRVEUpcuokFEURVG6jM5Xb6zRaKSfn5+v3l5RFKVXqqqqklLKXlNB8FnI+Pn5UVlZ6au3VxRF6ZWEENW+LkN79Jo0VBRFUXofFTKKoihKl/FZc1lXs9lsZGRkYLVafV2UHslkMhEfH49er/d1URRF6cP6bMhkZGRgNpsZNGgQQghfF6dHkVJSWFhIRkYGiYmJvi6Ooih9WJ9tLrNarYSFhamAaYYQgrCwMFXLUxSly/XZkAFUwLRC/W4URekOfTpkFEVRFN/qdX0yR3PKWf1jBvfPHEqAsdcVv1WBgYFUVFR4/LyiKN3rVPEp1h5di5SS6MDoRrdgU7BqIWhGr/uUziiu4o0tp7hiVBQTBoX6ujiK0m2klFjtVqpsVYT4haARqiGiO5wuPs2qw6tYdXgVe7L2tHicQWtoHDwB0cSYY84Jo6iAKPz0/We1k14XMklxFgD2Z5T26JB57LHHSEhI4Fe/+hUA//Vf/4XZbObuu+/mmmuuobi4GJvNxp///GeuueYaj84ppeTRRx/liy++QAjBk08+yaJFi8jOzmbRokWUlZVht9t59dVXmTJlCrfffjt79uxBCMHSpUv5zW9+05U/cr/mlE5sDhu1jtpzbjZn4+dr7DVU1FZQXltOeU05ZTVlZ7dryyivKae81v28e7vuOId0AOCn82N4+HBGhI9gZPjI+vuhYUMx6Uw+/m30fqklqaw65AqW3Vm7AZgQO4HnrniOhaMWEmwKJqcip/6WXZ7t2q50PU4tSWVnxk7yK/ORnHthyMemPsYzlz/TZeUXQswC/g5ogTellM802X8z8Jj7YQXwSynlPve+VKAccAB2KeX4zpSl14VMZJCJ6CATBzJKPH7NHz87xOGsMq+WY1RsEH+YN7rF/YsXL+aBBx6oD5kPP/yQDRs2YDKZWLNmDUFBQRQUFDBp0iTmz5/vUTX7448/Zu/evezbt4+CggImTJjA9OnTWblyJVdddRW/+93vcDgcVFVVsXfvXjIzMzl48CAAJSWe/756urKaMoxaI0adscveo7ymnPTSdNJK00grSTu7XZpGRlkG1bbqRsFR9+HfURqhwWwwE2QMwmw012/HmmPPPm8wYzaa8dP5kV6aTkpBCrsydvHvg/+u/yDTCA2JwYmMjBjJiLARrnt3AIX4hXjjV4NTOl0h2UIYnhOeDZ4vqymjsrYSk85EiF8IwaZggo3Brnv3rf75JreuDs+0krT6Gsv3md8DMD52PH+9/K8sHLWQxJDGw/2DTcGMCB/R6jntTjv5lfmNAimnIocJcRO67OcQQmiB5cAVQAawWwixVkp5uMFhp4EZUspiIcRs4A3gogb7L5VSFnijPL0uZACS4i3szyz1dTFadf7555OXl0dWVhb5+fmEhIQwcOBAbDYbTzzxBFu2bEGj0ZCZmUlubi7R0dFtnnPbtm3ceOONaLVaoqKimDFjBrt372bChAksXboUm83Gtddey7hx4xg8eDCnTp3i3nvv5eqrr+bKK6/shp/a+2rsNezN2cvOjJ3szNzJzoydpJakAhCgDyDEL4RQv9D6W4ipjcd+IQQaAsmvzG82QNJL00krSaPYWtyoHDqNjgFBA0gITmB6wnQC9YEYtIZmb3qtvsV9Bq0BvUZPoCGwPlCCjEH46fw63J5fZaviWOExjhQcISU/hZSCFI4UHOGrk19R46ipPy4yIJKR4SMZHjYcnUZHjaPGdbPXYLVbm92ucbgf288ea3PaPCpXc8FpNpqJDoym2lZNibWEtJI0SqwlFFuLqXXUtno+k85UHzhRAVHEBcURZ3bdYs2x9Y9jzDEYtAaPypheml5fY9mVuQuAC2Mu5NnLn2XhqIUMDhns0XlaotPoiDHHEGOO6dR52mkicEJKeQpACPEBcA1QHzJSyu0Njt8JxHdVYTwKGU+rT0KICbgKvEhK+ZG3CtlUcpyFrw7nUm61YTa1PWO9tRpHV1q4cCEfffQROTk5LF68GIB//etf5Ofn88MPP6DX6xk0aJDH81WkPLfaDTB9+nS2bNnC+vXrWbJkCY888gg///nP2bdvHxs3bmT58uV8+OGHvPXWW1772bqClJL00nRXoLhD5cfsH+s/fAYEDWBS/CTuvvBuHE4HRdVFFFmLKKouori6mKMFR13PVRc1+nD1hNlgJiE4gYGWgUyOn0yCxbWdEJxAgiWB6MBotBptV/zYneav92dc9DjGRY9r9LzD6eB0yen68DlScISUghQ+PvIxTunEqDVi0pkw6oyNtoOMQRh17sda4znHGXXGsyHpDo66+6Y1rvYEp9Vupbi6mBJryTm3Ymtxo+2cihx2nNlBZnlms+EU4R/RKITigtxBZI4j3D+crelbWXV4FTszdgKuYHlm5jPcMPqGTgdLN9AJIRp2Dr0hpXyjweM44EyDxxk0rqU0dTvwRYPHEvhSCCGB15ucu/2FbcexrVaf3FW0Z4GNnSmQJ5LiXf0yBzPLmHxeWFe/XYctXryYO++8k4KCAr799lsASktLiYyMRK/X880335CWlubx+aZPn87rr7/OrbfeSlFREVu2bOG5554jLS2NuLg47rzzTiorK/nxxx+ZM2cOBoOB66+/nvPOO4/bbrutwz+HUzo5WnCUHRk72HFmB/vz9uOv9yfcP5xwv3DC/MNc2/7hhPk12PYPw2wwt/hBU1lbyZ6sPY1qKTkVOYCrz2F87Hjuv+h+JsVP4qK4i4gLivO4zNW26vrAqbsVW4spqi6i1FpKZEBkfYAMtAzskyODtBotQ0KHMCR0CHOHzfV1cdpk0pna/a1fSklhdSFZ5VlklmWSWZ5JZlmm63G56/H3md+TX5V/zmsviLmAv8z8CzeMuoHzQs/z5o/S1drqJ2nuD7nZb6hCiEtxhcy0Bk9PlVJmCSEiga+EEEeklFs6WlhvNpfdC6wGuq6x0a2u8/9AZkmPDpnRo0dTXl5OXFwcMTGu/zg333wz8+bNY/z48YwbN44RI1pv021owYIF7Nixg7FjxyKE4K9//SvR0dG88847PPfcc+j1egIDA/nnP/9JZmYmv/jFL3A6nQD85S9/8fh9Sq2l7MrcxY4zO9iRsYNdmbsosbr6dEJMIZwfcz61jloO5R2ioKqAwupCnNLZ7Ln0Gn194NSFj7/en/25+9mfu7/+dUNDh3LF4CuYFD+JSfGTSIpMQq/t+Lpqfno/4vRx7QompfcRQtT/XSVHJbd4XK2jluzybDLLM8mpyCE5KpkhoUO6saTdKgMY0OBxPJDV9CAhRDLwJjBbSllY97yUMst9nyeEWIOr+a3DISNaaoJpUpjTQDGuNDyn+iSEiANWApcB/wDWNddcJoS4C7gLwGAwXFhT074mjYamPvM15w8M5uWbLmh2f0pKCiNHjuzw+fuDlJQUho8Y3qiWsiNjB4fzDyORCASjI0czOX6y6zZgMsPChp0zdNYpnZRaSymoKqgPnfrtKvd29dntspoyRkWMqg+UiXETCfcP99FvQVF6FyFElZQyoJX9OuAYMBPIBHYDN0kpDzU4ZiDwNfDzhv0zQogAQCOlLHdvfwU8JaXc0NHyelqTaav69ALwmJTS0VqTgzuc3gAICAhoO91akRxv4UAP7/zviaSUbE7dzLb0bXx55EsOrT1U38kdbApmUvwkfjb6Z0yOn8zEuIlYTJY2z6kRGkL8QgjxC2Fo2NCu/hEURWmFlNIuhFiGq+tCC7wlpTwkhLjHvf814PdAGPCK+zO7rgkuCljjfk4HrOxMwNSdxJNCt1V9Gg984C5YODBHCGGXUn7SmcK1JinewhcHcyitsmHxV8vVe+J44XHuXnc336R+g0AwxDKEhaMWtlpLURSl95FSfg583uS51xps3wHc0czrTgFjvVmWNkOmmerTlcBTTQqW2OD4Fbiay7osYACS44IBOJBZyrShzTe1SCn7XGduR9gcNp7f8Tx//PaPGLVGXr36VRaPXkx2arZqUlQUpUt5UpNptvrUpOrV7epn/meWNBsyJpOJwsLCfr/c/56sPdyx9g725e7jupHX8dLsl4gJjKGwsBCTSc0MVxSla7UZMi1Vn1oKFynlbZ0vVtss/noSwvw5kNF8v0x8fDwZGRnk5587dLE/qLJX8fLBl/nn8X8SZgzjxSkvcnn85ZRmlFJKaf2VMRVFUbpSr5zxXycpzsJP6c0vl6LX6/vtVR+/OvkVd395N6dLTnP3hXfzzOXPEGwK9nWxFEXph3p1L29yvIXMkmoKKzo+FLovKawq5NZPbuXK965Er9Xz7W3f8trc11TAKIriM706ZJIadP73Z1JK3j/wPiOXj2TlgZU8efGT7LtnH9MTpvu6aIqi9HO9urlsTFwQAAcySrlkeKSPS+Mb6aXp/HL9L/n8+OdMjJvIpnmbSIpK8nWxFEVRgF4eMmaTnsERAf2yJuNwOli+ezlPbHoCgBeueoFlE5f12EUcFUXpn3p1yIBrReZdp4t8XYxudTDvIHesvYNdmbuYNWQWr179KoOCB/m6WIqiKOfo1X0yAEnxwWSXWskr92y5/N7um9PfcMHrF3Cy+CT/uu5ffH7T5ypgFEXpsXp/TaZ+2f9SLhvRtycXllhL+PknPycxJJHvln6nFpVUFKXH6/U1mVExQWgE7G9hUmZf8uvPf01ORQ7vLXhPBYyiKL1Crw+ZAKOOIZGBLc787ys+OPgBKw+s5PfTf9+l1wdXFEXxpl4fMuCaL7M/s7TFyxP3dhllGfxy/S+ZFD+J3178W18XR1EUxWN9ImSS4y3kl9eQW9b3Zv47pZPbPrkNm8PGuwveRafp9d1oiqL0I30iZJLcnf/7M5pfx6w3e2nXS2w6vYm/XfW3vny5WEVR+qg+ETKjYoLQakSfm5R5KO8Qj/3nMeYOm8sdF5xzfSFFUZQer0+EjEmvZViUuU+NMKt11HLLmlsIMgbx5rw3+/U1cRRF6b36RMiAa+b/gT7U+f+Hb/7A3py9vDn/TaICo3xdHEVRlA7pMyGTFG+hqLKWzJJqXxel07ambeXZ757ljvPvYP7w+b4ujqIoSof1mZCpm/nf2+fLlNWU1c/q/9+r/tfXxVEURemUPhMyw6PN6LWC/b288//+DfeTXprOuwvexWw0+7o4iqIondJnQsao0zIiOqhX12Q+TvmYFXtX8Ntpv2XKgCm+Lo6iKEqneRQyQohUIcQBIcReIcSeZvbfLITY775tF0KM9X5R25YUb2F/Rkmv7PzPLs/mrs/u4sKYC/nDjD/4ujiKoihe0Z6azKVSynFSyvHN7DsNzJBSJgN/At7wSunaKTnOQpnVTnpRlS/evsOklNy+9nYqbZW8u+Bd9Fq9r4ukKEovJoSYJYQ4KoQ4IYR4vJn9LVYM2npte3mluUxKuV1KWex+uBOI98Z52+vszP/e1WT22p7X+OLEFzx3xXOMjBjp6+IoitKLCSG0wHJgNjAKuFEIMarJYc1WDDx8bbt4GjIS+FII8YMQ4q42jr0d+KK5HUKIu4QQe4QQe+x2e3vK6ZFhUWYMOk2vmvl/tOAoD335EFeddxW/nvBrXxdHUZTebyJwQkp5SkpZC3wAXNPwgFYqBm2+tr08XW1xqpQySwgRCXwlhDgipdzS9CAhxKW4QmZacyeRUr6BOzEDAgK83nGi12oYGRPUa9YwszlsLFmzBD+9H29d85aa1a8oiid0TfrG33B/ttaJA840eJwBXNTK+RpWDNr72jZ5FDJSyiz3fZ4QYg2utGsUMkKIZOBNYLaUsrAzheqM5DgLa37KxOmUaDQ9+0P7z1v+zO6s3ay6YRWx5lhfF0dRlN7B3kLfeJ3mPvia/VLfTMXA49d6qs3mMiFEgBDCXLcNXAkcbHLMQOBjYImU8lhnCtRZSfEWKmrsnC6s9GUx2rQzYydPb32aJclLWDhqoa+LoyhK35EBDGjwOB7IanpQg4rBNQ0qBh69tj086ZOJArYJIfYB3wPrpZQbhBD3CCHucR/zeyAMeKWlYc7dpTfM/K+orWDJmiXEBcXx0uyXfF0cRVH6lt3AUCFEohDCACwG1jY8oJWKQZuvba82m8uklKeAc+a9SClfa7B9B9Aj1qIfEhGISa9hf0Yp154f5+viNOuhjQ9xsugk39z6DRaTxdfFURSlD5FS2oUQy4CNgBZ4S0p5qK5S4P7sblgxAHcTXEuv7Ux5hK8mLgYEBMjKyq5p0rr+1e1oBKy6p+fNmt+ZsZPJ/5jMI1Me4a9X/NXXxVEUpZcRQlRJKQN8XQ5P9ZllZRpKirNwMLMMh7PnzfxfeWAlJp2J38/4va+LoiiK0uX6ZMgkx1uotjk4mV/h66I04pROPk75mKvOu4pAQ6Cvi6MoitLl+mzIQM+b+b8rYxeZ5ZlqNJmiKP1GnwyZxPBAAgxaDvSwSZmrU1aj1+iZO2yur4uiKIrSLfpkyGg1gtFxlh51bRkpJatTVnP54MsJNgX7ujiKoijdok+GDLhm/h/OKsPmcPq6KAD8lPMTqSWpXD/yel8XRVEUpdv02ZBJirdQY3dyPLdndP6vPrwardByzYhOrTWnKIrSq/TZkEmOdzVJHcj0fb9MXVPZjEEzCPcP93VxFEVRuk2fDZmEUH/MJl2PGGF2OP8wRwuPqqYyRVH6nT4bMhqNICnO0iOuLbM6ZTUCwYIRC3xdFEVRlG7VZ0MGXP0yKdll1NgdPi3H6pTVTBkwhRhzjE/LoSiK0t36dMgkxwVjc0iO5fiu8/944XH25+5XEzAVRemX+nbI1M3892Hn/+qU1QBcN/I6n5VBURTFV/p0yMSH+BHsr/fptWVWp6xmQuwEBloG+qwMiqIovtKnQ0YIV+e/r0aYpZWksSdrjxpVpihKv9WnQwZcTWbHcsux2rq/8//jlI8BuH6UChlFUfqnPh8ySXHB2J2SlOyybn/v1SmrSY5KZkjokG5/b0VRlJ6gz4dMXed/d8+XyS7PZvuZ7aqpTFGUfq3Ph0yMxUR4oKHb+2XWHFmDRKqQURSlX/MoZIQQqUKIA0KIvUKIPc3sF0KIF4UQJ4QQ+4UQF3i/qB1T1/nf3SPMVqesZnjYcEZFjOrW91UURelJ2lOTuVRKOU5KOb6ZfbOBoe7bXcCr3iictyTFB3M8r5yqWnu3vF9BVQHfpn7L9SOvRwjRLe+pKIrSE3mruewa4J/SZScQLIToMWuoJMdZcEo4nNU9nf+fHvkUh3SoWf6KovR7noaMBL4UQvwghLirmf1xwJkGjzPczzUihLhLCLFHCLHHbu+eWgW41jADuq1f5qOUj0gMTmRc9LhueT9FUZSeytOQmSqlvABXs9ivhRDTm+xvrk1InvOElG9IKcdLKcfrdLp2FrXjooJMRAUZu2WEWYm1hE2nNqmmMkVRFDwMGSlllvs+D1gDTGxySAYwoMHjeCDLGwX0lqS4YPZndP0aZp8d/Qyb06YmYCqK4jNCiFlCiKPuwViPN7N/hBBihxCiRgjxcJN9rQ70aq82Q0YIESCEMNdtA1cCB5scthb4uXuU2SSgVEqZ3dnCeVNyvIVTBZWUW21d+j6rU1YTZ45jYlzTHFYURel6QggtsBxXy9Mo4EYhRNNhrkXAfcD/tHCa1gZ6tYsnNZkoYJsQYh/wPbBeSrlBCHGPEOIe9zGfA6eAE8D/Ab/qbMG8LSnegpRwqAs7/ytqK9h4ciPXjbwOjejzU5AURemZJgInpJSnpJS1wAe4BmfVk1LmSSl3A137rRtos2NESnkKGNvM86812JbAr71bNO9KinPP/M8oZdLgsC55j8+Pf47VblUTMBVF6Uq6Js1Yb0gp32jwuLmBWBe14/x1A70k8HqTc7db9/W++1h4oJG4YL8u7fxfnbKayIBIpg2c1mXvoShKv2dvoxnLo4FYrZgqpcwSQkQCXwkhjkgpt7SviGf1qzadpDhLl4VMta2a9cfWc+3wa9FqtF3yHoqiKB7o1EAsDwZ6tUv/Cpl4C6cLKimt9n4z5Jcnv6TSVqkmYCqK4mu7gaFCiEQhhAFYjGtwVps8HOjVLv2muQzOrsh8KLOUKUPCvXru1SmrCTGFcMmgS7x6XkVRlPaQUtqFEMuAjYAWeEtKeahuoJaU8jUhRDSwBwgCnEKIB3CNRAsH1rjn+OmAlVLKDZ0pT78KmbrO//1eDplaRy1rj65lwcgF6LV6r51XURSlI6SUn+Ma9dvwuYaDtXJwNaM1VUYzA706o181lwX7GxgY6u/1FZk3ndpEaU2pGlWmKIrSRL+qyYCrNrM/07sz/1enrMZsMHPF4Cu8et6+qHxvOXn/ysN/lD+hV4ZijDP6ukiKonSh/hcy8RbWH8imqLKW0ABDp89nd9r55MgnzB02F6NOfWC2pGxPGWl/SqNwbaFrgKV7QGXAmABCrgoh9KpQLBdb0JrUyDxF6Uv6XchMGBQCwNbj+Vwz7pyFotttS9oWCqsLVVNZC8p2lZH6VCpFnxehC9Yx6I+DiLs3jpozNRRtLKJoYxGZL2WS8XwGGpMGywwLoVeFEnpVKP4j/dUio4rSy/W7kDl/QAgxFhOf7cvySsisPrwaP50fs4bM8kLp+o7S7aWkPpVK8cZidKE6Ep9OJG5ZHLog15+cPkRPYHIgAx8ZiKPSQcm3JfWhc/LBk5zkJMZ4Y30tJ+TyEPQhalCFovQ2/S5kNBrB1UkxvLMjldIqGxb/jn9wOaWTNUfWMHvobAIMAd4rZC9WsqWE1KdSKdlUgj5Cz+BnBxP7y1h05pb/1LQBWsLmhBE2x7XcjzXNStGXrsDJ/yifnH/kgAbME8yuWs6sUIIuCkJouq+WI6WkfE85Wa9nkf9hPkIn0EfoMUQa0Efo62+GiOYfawz9aoyNotQTrmXHul9AQICsrKz0yXvvO1PCNcu/46/XJ/OzCQPafkELvkv/jmlvT+Nf1/2Lm5Ju8mIJexcpJSWbS0j9Yyql35aij9Iz8NGBxN4dizagc30sTruT8u/L62s55bvLwQmGOANRN0YReVMkgeMCu6xZzV5uJ29lHlmvZ1HxUwUafw0RN0SgDdRiy7dhy7dRm1fr2i6wgbP582iDtPWhY4gxMPDxgQRNDOqSMit9mxCiSkrZa77V9suQkVIy47nNJIT58+7t7Vk3rrEHNz7I8t3LyX8knyBj//vAkFJS/J9i0p5Ko3RbKYZYAwMfG0jMnTFo/bqmA99WZKNoQxF57+dRtKEIaZf4j/Qn8qZIom6Mwu88P6+8T/lPrlpL3r/ycFQ4CEgOIPbuWKJujkJnab5WJp0Se7Gd2vza+gCy5dvOeVxxoAJ7iZ0Rb48g6sYor5RX6T9UyHjIlyED8NzGI7z27Sl2PTGT8MD2jwqTUpL490SSopL47MbPuqCEXUNKSeWhSpCgMWnQGDWue/dN6EWbtQIpJUUbikh7Ko2ynWUY440MfHwg0bdHd+voMFuhjfyP8sldmUvpFtfcJ/NFZqJuiiLiZxEYo9v37+qodJD3b1etpfz7cjQmDRGLIoi9J9bVPOel2lJtQS2HrjtE6dZSEn6fwKA/DOrWpj+ld1Mh4yFfh8yRnDJmvbCVP107hiWTEtr9+j1Ze5jwfxN4+5q3uW3cbd4voJfZy+3k/jOXzJczqTpS1fKBgnODxygaPbYX2qk8WIlxoJGEJxKIvi0ajdG3fQ7WdCt5H+SRuzKXyn2VoIGQy0OIuimK8AXh9QMOmlNxsILs17PJeTcHR6kD/1H+rlrLkqguG2zgrHVy7J5j5LydQ8QNEYxYMQKtvxq+rbRNhYyHfB0yUkqu/NsWQgIMfHj35Ha//vH/PM7zO54n9+FcQv1Cu6CE3lF5pJKs5VnkvJODo9yBeYKZmDtj0IfqcVqdrluN8+x2c4+bPAcQvSSaqCVRPbJDu/JQJbnv55K3Mg/raSsak4awuWFE3hRJ2JwwNEYNjmoH+R/lk/V6FmXflSEMgogbIoi9OxbLNEu3DJ2WUpLxvxmcfOQkgRcEkvRpkpqc2sVsxTbyP8zHmGAk5PIQNLqe9/fbFhUyHvJ1yAD8/T/HeWHTMbY/fhkxFs/b8qWUDHt5GInBiXy55MsuLGHHSIekcH0hmS9nUvxVMcIgiFwUSdyyuH7V2SylpGxXGXkr88j7dx62PBtai5aQy0Io2VyCvdiO3zA/Yu+KJerWKAzhnZ+c2xEF6wpIuTEFrVnLmLVjCBrff/6Nukv1qWoyXsgg+61snJWuL0r6SD2RN0YSdUsU5gvNvWZOlgoZD/WEkDmZX8HM57/lyatHcsfFgz1+3f7c/Yx9bSyvXf0ad4+/uwtL2D62QhvZ/8gm85VMatJqMMQZiPtlHDF3xmCI9M0HaE/htDsp2VRC7spcir8qxnKxhdh7Ygm+JLhHfLhUHKjgwLwD2PJsjHhnBJE3RPq6SH1C6Y5Szjx/hoI1BQitIPLGSOLvi8d6xkrue7kUflaIrJX4Dfcj6pYoom6Owi/RO4NHuooKGQ/1hJABuPrFrei0Gj799VSPX/OHb/7An7b8ieyHsokK9P3ooPKfysl8KZO89/NwWp0EXxJM3LI4wq4J65XNAf1VbV4tBxccpGx7GYOeGkTCkwk9IgA7Qjok9hI79hI7tmKba7vYfSuzEzAqAMvFllbnT3XmvQs+LeDM/5yhbEcZumAdsffEEndvHMbYxs2RtmIb+avzyX0vl9JvXYNHgqYGEXVLFJE3RKIP63kTgFXIeKinhMxr357kmS+OsPXRSxkQ6u/Ra8a8MoYw/zC+ve3bDr2n0+5EaNsexdXqOWqd5K/OJxxVB7wAACAASURBVPPlTMq2l6Hx1xC1JIq4X8cRmBTY4fMqvuWwOjh21zFy380lcnEkw98a3mXDwdsinRJ7mR17oR1bkQ17kR1boc21XRcY7vBoFCQldhxljjbPL3QC80QzITNDCL4sGMtkS6cGkDgqHWS/nU3GCxlYT1oxJZqI/0080b+IRhfYdphZ06zkvp9L7ru5VB2uQugFoXNCibolirC5Ye0eOWkrsWE9ZaX6ZLXr/lR1/XbMHTEkPNH+AUfQh0NGCKHFdZGbTCnl3Cb7LMB7wEBcqwj8j5Ty7dbO11NC5kxRFRf/9RsenTWcX10ypM3jjxYcZcTyEfx91t+576L7PH6f6tPVFKwpoGBNAaXfub4xafw0aP21aPw0aPw1aP20aPw1jZ7X+p/7nL3UTu47udTm1OI3xI/YX8cSfVs0+uCe961LaT8pJenPpnP6idOYJ5gZ88kYjDHeGRDgqHRQtruM2uzaxqHh3rYXuQLFVugKkpYmlwJoAjToQ/ToQnTognWue/e2PkR/znN199oALRU/VlD8dTHFm4rrJ9hq/DRYplkIviyYkJkhmC8wI7RtfxGrya4h86VMsl7Lwl5sJ2hyEAMeGkD4teEevb4pKSUV+yrIfc81eKQ2uxatRUvEwgiibokieHowQiNw2p3UZNScEyR1j+3F9kbn1YfrMZ1nwm+wHxHXRxBxfUS7ywZ9O2QeBMYDQc2EzBOARUr5mBAiAjgKREspa1s6X08JGYDrXvmOapuTL+6/uM1jX/7+Ze794l5O3XeKxJDEFo+TUlJ5oJKCNQXkr8l3DasFApIDCJ0VisbgGuHkrHLirHbiqHKcvW/uuWonzqqz/+ND54QStyyO0KtC1RyLPir/k3xSbk5BH6pnzNoxmM83t/scjkoHpdtLKdlcQsnmEsq/L0faG/+f1wZp0Yfp0YXq0IfqG23rQnXn7nMHh0bvnaZYe6mdki0lFG8qpmRTCZUHXf9XtBYtwZe4AidkZsg5C6ZWHKjgzPNnyFuZh7RLwheEM+ChAVimWLxSLnA1vRV/U0zue7kUrC7AUeHAEGdA66fFmmpt9LsUOoFpkAnTYBN+5/m57gf7ubYTTa0Oo2+PPhkyQoh44B3gaeDBZkLmt8AA4NfAIOArYJiUssXvQT0pZN7adpqn1h3mPw9OZ0hk6/+RF320iO1ntpP+QPo5zV3SKSnbWVYfLNaTVhAQNDmI8AXhRCyI6NSMdCklzhonOOj0ci1K71C+t5yD8w5iK7Ix8r2RRCxo/dtvi6GihaAJQQRfEoxlugVToskVGMHeCwtvqc2tpfgbV+AUf12M9ZQVAEO0geDLggmaFEThukKKvyxG468hZmkM8Q/Ee221h5Y4qhwUrC0g/6N8hFY0ChLTYBPGeGO39IH21ZD5CPgLYAYebiZkzMBaYIT7mEVSyvXNnOcu4C4Ag8FwYU1NTad/AG/ILbMy6S+buO+yofzmimEtHielJP5v8cxImMHK61cCrr6Rkm9KyF+TT+GnhdTm1CL0guDLgolYEEHYNWHtnnmuKA3V5NRw8NqDlO8qJ/G/Exn4+MD6Lzj2Cjtl28vOhspuV6gIncA8wUzwJcEEXxJM0JQgj/oleqLq1Or6wCneVIwt14Yh2kDcfXHE3h2LPrR/NRP3uZARQswF5kgpfyWEuITmQ2YhMBV4EDgPV01mrJSyrKXz9qSaDMDiN3aQV17DpgdntNghf6r4FOe9eB6vXvoq1+df7wqWdYU4Sh1oAjSEzQ4jfEE4oXNCVf+I4lWOagdHbz9K3vt5RC6OxJRo6rOh0hopJdbTVoxxRp+vMuErvS1kPPkrnArMF0LMAUxAkBDiPSnlLQ2O+QXwjHQl1gkhxGlctZrvvV7iLjJvbCy/W3OQlOxyRsU2Pxlua9pWLjh5ASP+OoJDVYfQheqIWBBB+IJwQq4I8dkoIKXv0/ppGfmvkfiP8if1/6XWh8qARwb06VBpSgiB3+CePY9FaaxdQ5hbqcm8CuRKKf9LCBEF/IirJlPQ0rl6Wk2mqLKWCU//h7umD+axWSOaPea+f9zH5fddTvSgaIa+PBTLxRY1D0XpdtYMq2u0Vj8IFeVcva0m0+FPSCHEPUKIe9wP/wRMEUIcADYBj7UWMD1RaICBaUPC+WxfFs0Fr7PWydinx2Jymhjz8RhCLu2d6x4pvZ8p3qQCRuk12vWXKqXcDGx2b7/W4Pks4EpvFswX5o2N5eFV+9h7poTzB4Y02nfwwYOcd/o8TvzxBP7DPZu0qSiK0t+pr+INXDk6CoNWw2f7shs9n/dRHkXLi1h90WpG/2K0j0qnKIrS+6iQaSDIpGfG8AjWH8jC6XQ1mVUdr+Lo0qMUDS9ixewVXBh7oY9LqSiK0johxCwhxFEhxAkhxOPN7B8hhNghhKgRQjzcnte2lwqZJuaNjSW3rIbdqUU4qhwcWngIoRe8ctsrXJhwIQZt/17NWFGUns29BNhyYDYwCrhRCDGqyWFFwH3A/3Tgte2iQqaJy0dG4qfX8tn+LI4vO07lgUoGrRjEN7XfcPHAtpedURRF8bGJwAkp5Sn30l4fANc0PEBKmSel3A3Y2vva9lIh04S/QcfMkZEUvJNLzts5JDyZQMqoFJzSycUJKmQURfE5nRBiT4PbXU32xwFnGjzOcD/nic68tllqHGQz5htDYV0pTA5g0B8G8faWt9EIDZPj23+ZZkVRFC+zSynHt7K/uSVLPJ0Q2ZnXNkvVZJqwl9oxP5ZDlR98tVSH0Aq2pm/l/OjzMRvbvwquoihKN8vAtWBxnXggqxte2ywVMg1IKTnyiyPUpFo5+LCZzzLyKK+pZmfGTqYNnObr4imKonhiNzBUCJEohDAAi3EtYNzVr22Wai5rIOOFDArWFHDe8+chrzby5tu5vP3911jtVtXpryhKryCltAshlgEbAS3wlpTyUN0KLVLK14QQ0bguQhkEOIUQDwCjpJRlzb22M+Xp95dfrlP6XSl7L9lL2LwwRq8ejd0pmfD0fwgMW893BS+S81AOUYFRvi6moij9XL9Zu6wvqc2r5dCiQxgTjIx4ewRCCPRaDbPHxLAvfxdDQ4epgFEURemAfh8y0iFJuTkFe6Gd0R+NRmc524J4dXIUVRwi0axm+SuKonREv++TSX0qleL/FDP8zeGYxzUePWYOzMUpKqitavlqmYqiKErL+nVNpnBDIWl/SiP6tmiil0afs3/7mW0ApGYNpNzadGKsoiiK0pZ+GzLWM1ZSbkkhYEwAQ5cPbfaSy1vTtxLuF4XTHsl/UnJ9UEpFUZTerV+GjLPWyeGfHUbWSkZ/NBqtf/OXTd6Wvo1LE2cQH+x/zvL/iqIoStv6ZcicfPQkZTvLGP7WcPyHNX8BsrSSNM6UnWF6wsXMTY5hy7F8Sqpqu7mkiqIovVu/C5ms/8si8++ZxD8QT+TCyBaP25q+FYBpA6cxb2wsdqdkw8Gc7iqmoihKn9CvQib7H9kcu+sYoXNCGfzs4FaP3Zq2lSBjEEmRSYyODWJQmD/r9qsmM0VRlPboNyGT/VY2R+88SujsUEavHo3G0PqPvjV9K1MHTEWr0SKEYN7YWLafLCC/vKabSqwoitL7eRwyQgitEOInIcS6FvZfIoTYK4Q4JIT41ntF7Lycd3I4esdRQq4MYfTHo9Gamu/or1NQVUBKQUqj9crmjY3FKeGLg6o2oyiK4qn21GTuB1Ka2yGECAZeAeZLKUcDN3ihbF6R824OR35xhJDLQxjzyZg2Awbgu/TvABpdpGxYlJnhUWY+29epVa8VRVH6FY9CRggRD1wNvNnCITcBH0sp08F1aU/vFK9zct7L4citRwiZGcKYTz0LGHA1lRm0BsbHNr4u0LyxMexOLSarpLoriqsoitLneFqTeQF4FHC2sH8YECKE2CyE+EEI8fPmDhJC3FV3yVC73d6B4noud2UuR249QvClwa6A8fMsYMAVMhPjJmLSmRo9Pzc5FoD1agCAoiiKR9oMGSHEXCBPSvlDK4fpgAtx1XauAv6fEOKcBb+klG9IKcdLKcfrdF23bFruB7mkLEkheEYwSZ8ltTjZsjmVtZX8mP1js9ePGRQeQFKchXX7VZOZoiiKJzypyUwF5gshUoEPgMuEEO81OSYD2CClrJRSFgBbgLFeLamH8v6dR8rNKVgutrQ7YAB2Ze7C7rS3eJGyeWNj2JdRytGccm8UV1EUpU9rM2SklL+VUsZLKQfhuhTn11LKW5oc9ilwsRBCJ4TwBy6ihUECXSlvVR6Hbz6MZZqF5PXJaAPaFzDgmh8jEEwZMKXZ/QsvHIDFT8//+/Qgvrrgm6IoSm/R4TarhpfylFKmCCE2APtx9du8KaU86KUyeiR/dT6HbzyMZbKFpPVJHQoYcPXHJEclYzFZmt0fGmDgiTkjeGz1AVb9kMHPxg/oTLEVpcez2WxkZGRgtVp9XZQ+zWQyER8fj16v93VRvKpPXH45f00+h392GPNEM8kbktGZO5adNoeN4GeDWTpuKS/NeanF45xOyaI3dnAir4JND11CaICho0VXlB7v9OnTmM1mwsLCml2tXOk8KSWFhYWUl5eTmJjY6rHq8svdrODTAlfATDCT/EXHAwbgp5yfqLJVNZof0xyNRvD0giTKrXaeXt/trYKK0q2sVqsKmC4mhCAsLKxP1hZ7dcgUrC3g0A2HMI9312CCOjdibVu66yJlLXX6NzQsyszdMwaz+scMdpws7NT7KkpPpwKm6/XV33GvDZmCdQUcWniIwPMDvRIw4OqPOS/kPGLMMR4df+9lQxkY6s/v1hygxu7o9PsriqL0Nb0yZAo/L+TQ9YcIHBdI8sZkdJbOB4yUkm3p25g2cJrHrzHptfzp2jGcKqjktc2nOl0GRVGaJ4RgyZIl9Y/tdjsRERHMnTsXgBUrVhAREcG4cePqb4cPHyY1NRUhBC+9dLaPddmyZaxYsaK7f4R+q9eFTNGXRRxccJDA5ECSv0xGH+ydkRhHCo5QUFXgUVNZQzOGRTBvbCzLN5/gVH6FV8qiKEpjAQEBHDx4kOpq15JOX331FXFxcY2OWbRoEXv37q2/jRo1CoDIyEj+/ve/U1urLjroC70uZEyJJkJnhXo1YODsRcra6vRvzv+bOxKjTsOTn6i5M4rSVWbPns369esBeP/997nxxhs9el1ERAQzZ87knXfe6cri9ShCiFlCiKNCiBNCiMeb2S+EEC+69+8XQlzQYF+qEOKAe1X9PZ0tS9et7dJF/If6k/RpktfPuy19G5EBkQwNHdru10aaTTw2awRPfnKQT/ZmsuD8eK+XT1F6ggc2PMDenL1ePee46HG8MOuFNo9bvHgxTz31FHPnzmX//v0sXbqUrVu31u//97//zbZt2+of79ixo3778ccfZ/bs2SxdutSrZe+JhBBaYDlwBa7VWHYLIdZKKQ83OGw2MNR9uwh41X1f51L36i2d1utCpqtsTd/KxQMv7vAIj5smDmT1jxn8eV0Klw6PJNhfzZ1pjdNpx+msxumswuGou6/C6azC6ayu33bdVzfa53TaCAxMwmKZhp/fsD47KkdpLDk5mdTUVN5//33mzJlzzv5Fixbx8ssvN/vaxMREJk6cyMqVK7u6mD3BROCElPIUgBDiA+AaoGHIXAP8U7qaXnYKIYKFEDFSSq+v/qtCBsgoyyC1JJX7L7q/w+fQaAT/vSCJuS9t45kvjvDM9cleLGHvV1V1gqKiLygq2kBp6RYcjo71X2k0/oDA6XRN5NXrw7FYptXfAgPPR6NRAd9VPKlxdKX58+fz8MMPs3nzZgoL2zd14IknnmDhwoVMnz69i0rXbXRNmrHekFK+0eBxHHCmweMMGtdSWjomDsgGJPClEEICrzc5d/sL25kX9xVb09z9Me3s9G9qZEwQd0xL5PUtp7j+wngmDAr1RvEakVJSXX0SKWsADa6aseteCA3gum/4/Ln7dWg0XftP73BUUVLyDUVFGygs/AKr9SQAfn5DiIpagsEQjUbjj1br7773a/RYo/E7Z59GY0IIgZSSqqqjlJZuo6zsO0pLt1FQ8AkAGo0fQUEX1YdOUNBkdLqgLv1Zle6zdOlSLBYLSUlJbN68uV2vHTFiBKNGjWLdunVMnDixawrYPexSyvGt7G+uat+0s7i1Y6ZKKbOEEJHAV0KII1LKLR0pKKiQAVxNZYGGQMZGd37h6PsvH8q6/dk88fEB1t93MQad98ZWVFef5vjxeykqWt/pcxmNCQQEjMTffxT+/iPd2yPR6zsWjK4P/iMUFW2gqOgLSkq2IGUNGo0fwcGXER//AKGhs/D3H9LpsgshCAgYQUDACGJj7wCgpianPnBKS7eRlvYXwAFoCAxMbhA6UzGZzvaZORxW7Pbi+pvNVozdXtRgu+G+Ivd2GTqdGZ0uDL0+HL2+7r7h9tnndLpgd9grnRUfH8/99zff4tC0T+aVV14hNja20TG/+93vOP/889v9vlJKnM4q7PYS7PYyhNCg0ZjcNyNC1N33iKbbDKDhoorxQNPrk7R4jJSy7j5PCLEGV/Nbh0OmT6xd1lnJryYTY45h4y0bvXK+TSm53P7OHh65aji/vrTzH6pOZy1nzjxPWtqfAA0JCU/g5zcUcCKlAymdgOteSkeT7cb3UjpxOquprj5OZeVhqquP4nSeXcpCr49qNnwMhphz/gPZ7eWUlHxNYaGrGaymJg0Af/+RhIbOJjR0FhbLxWi1jS/+1h3s9grKy3fVh05p6Y76JjaDIQ6Q2O1FjX725mi1Qej1oeh0IQ1uQTgcFdhsBdhshe77AqRsaYisQKcLbRA8EQQGjiUoaDJBQZPQ64O9+8N7WUpKCiNHjuzUOVx9cJU4HBU4HJVotYHo9eE9umlTSjt2exl2eykORylSui60qNG4lg1z/e00nIQtEMKIRmNsEEB14aP3KIA8+V23tXaZEEIHHANmApnAbuAmKeWhBsdcDSwD5uBqSntRSjlRCBEAaKSU5e7tr4CnpJQb2ix8C/p9Taa4upiDeQf52eifee2cM0dGMXtMNC9uOs7c5BgSwjq+ll1x8WaOH/8VVVUphIdfx5AhL2AyeW/lZykdWK1pVFWlUFl5mKqqFKqqUsjNfQ+Ho6z+OK3WUh84RmMcpaWuWoOUNrTaQEJCLich4beEhs7CZErwWvk6SqcLJCRkJiEhMwHXh1xl5T5KS7dRXv4DQhjQ688GR9Mg0etD0GotHjcrSilxOCrrA8duLzwnhOq2q6uPU1j4GXUXmvX3H+kOnMlYLJPx9x/pbtrsnaSUSFnrDhTXzek8e8lyjcZEbW0WtbXZ7t91JFptgM9rAa7airU+VFz9hhLQodMFodNZzvmbcIWnFafTipRWnM4anE4rNlsZjVuoNI2CR6sNQqcL7Kqfwy6EWAZsBLTAW1LKQw1Xzgc+xxUwJ4Aq4Bful0cBa9z/FjpgZWcCBlRNhnXH1jHv/XlsvnUzMwbN8Np5c0qtXP6/33JBQgjv/GJCu/8D1dbmcfLkw+TmvovJlMjQoS8TFnbuiJquIqWktjbbHT4pVFUdrt+22XIJCEgmNHQWoaGzsVim9OhvpD2R3V5Oefluysp2UFq6g7KyHdjtRYAr0IOCLqoPHbP5Ip/Wdtr6dl1XO24YKlLa3Hs1aLWBDW4BCKHF4bBis+VhsxUCDjQafwyGKHS6kG4NWCmdOBzl2O2l2O2l7r5OV9+eK1SCOxSAdUFbFzpnbzVIWYPBEIPRGHfO67xRk+lp+n1NZmvaVvQaPRPjvNsRGG0x8dCVw/jjZ4dZtz+beWNj234Rrj/67Oz/49Spx3E4Khk48AkSEn6HVuvv1fK1RQiB0RiL0RhbXxuo43TWoNEYu7U8fY1OZyYk5DJCQi4D6gZ0HG8UOq7mUScg6ms7Fouric1kOs8nzZCuD09Hg6YvV/NXXa1MCANarbk+VDQav2Y/oLVaE1rtQIzGOHcNLw+r9TRCnEGvj0Cvj+iSLy51H/6uZrASHI5yd9k1aLVmdLpodDpLp99biLNNZ9B44Imrebv/TNru9zWZKf9wXQFz++3bvX5uh1Ny7fLvyC61sumhGVj8Wl+hoLz8J44d+yXl5bsIDr6EoUNfISCgc23hSu/lqu18Xx86ZWU762s7ADpdCAaD64tA4/uYBo+jPfpCIKWrj6qmJpva2rO3mposqquvZ8iQSJxOm7uG4qx/nWv0X2CDUOnYh7OrubGc2tpcHI5SXP1Ywej1UR1uSnMFig2Ho9I958p1X9e3IoQBnS7YXWMx94gmSlWT6WOqbdXsydrDbyb9pkvOr9UI/nJdEvNf3sZzG4/w52ubX6nAbi/j9Onfk5n5Enp9OCNGvEtU1M0+b6NWfMtV2znbr+Sq7RyjrOx7amrS3YGQRU1NFlVVR6itza7/AG1Irw/HYIh1N9HEYjBEYbeXNQiSbPdrzx24oNWaMZvnAcL9Ya9HCD1arX9905c3CCHc/R5BOJ011Nbmufu2itFo/NHrI9HrQ1sMgrOBUuWuZbnuG/4+XMPig91lN9cPiVe6Vr8Ome8zv8fmtHVovTJPjYmzcNuURN7efprrLojngoEh9fuklOTnr+LEid9QW5tNbOw9JCY+jV4f0soZlf5KCIG//3D8/Yc3u19KJzZbATU1We4AyXJvn72vrDxIbW0OOl1QfegEB0/HYIhxP45xb7tqRFptACkpKS2+Z1fQaIyYTAMwGmOx2Yqw2XKpqUmltjajvikNaBIoVQ36geoCxYJWG1A//6on1FT6o34dMnWLYk4dMLVL3+fBK4fxxUHX3JnP7p2GXquhquoEx48vo7h4I4GB5zNmzBqCgnr1BDHFx4TQYDBEYjBEAuNaPE5K2Su+wQuhxWCI4NFHn2bAgGh+9auF1NZmM2fOQuLjo3j55ScB+N3vXiYuLo709Fy+/XY7QmgxmUx8+OGHJCZGMmvWLLKzs7Hb7Vx88cUsX74crVbLihUreOSRR+pXc162bBl33HEHaWlpXHfddTgcDmw2G/feey/33HOPL38VvVq/D5kxkWMI8evamkOgUccf5o3mwQ++ZeW2T5k+cB/p6c+g0RgYMuTvxMb+qstn4CtKnd4QMA1NnTqVVatW8dBDj2O3V1NcXEllZQZ+fiPQav3YvfsoYWGJ5OaWsH//QTQaDRkZGQQEuLotPvzwQ4KCgpBSsnDhQlatWsXixYuB5tc7i4mJYfv27RiNRioqKhgzZgzz588/Z2Kn4hmPP9ncK3vuATKllHNbOGYCsBNYJKX8yDtF7Bp2p53tZ7azJHlJ2we3g9NZQ3X1SaqqjlJdfYyqqmNUVx/DXHWMVy7PAwlpaRARsYghQ/4Xo1H94SpKa6ZOncpvfuPqN01JOUFS0vlkZ2dTXm7D319PSkoKCxYsICYmBo3G1SQWH392VYegINfoLrvdTm1tbZshazCcHbxQU1OD0+ls5WilLe35+nw/kELT8Xhu7hB6FtcEoB5vf+5+KmorOrRemZROrNb0RiFSd2+1ptFw9I1eH4W//3DCw+djE4n8eUMNUaHJvDDjul73jVJRjj9wnIq93r04X+C4QIa+0PIlNmJjY9HpdKSnp7N9+3YmT55MZmYmO3bswGKxkJyczE033cS0adPYunUrM2fO5JZbbmm0fMxVV13F999/z+zZs1m4cGH986tXr2bLli0MGzaMv/3tbwwY4JrofObMGa6++mpOnDjBc889p2oxneBRyAgh4oGrgaeBB1s47F5gNTDBO0XrWvWLYraj099qzeDMmWfJzn4Lp7Oq/nmtNhA/v2EEBU0iKurn+PsPw89vGP7+Q9HpLI3OMTPnFE9/nkLMhqM8Nmu4ChpF8cDUqVPZvn0727dv58EHHyQzM5Pt27djsViYMmUK8fHxHD16lK+//pqvv/6amTNnsmrVKmbOdI3M27hxI1arlZtvvpmvv/6aK664gnnz5nHjjTdiNBp57bXXuPXWW/n6668BGDBgAPv37ycrK4trr72WhQsXEhUV5ctfQa/laU3mBeBRwNzcTiFEHLAAuIxWQkYIcRdwFzSukvrC1vStDAoeRHxQ2xcYs1rTSE9/huzstwAnkZE3Y7FMrQ8TgyHa47BYOi2RUwWVvPbtSQoravjLdUnotGrUi9I7tFbj6EpTpkxh+/btHDhwgDFjxjBgwACef/55goKC6i9EZjQamT17NrNnzyYqKopPPvmkPmQATCYT8+fP59NPP+WKK64gLCysft+dd97JY489ds77xsbGMnr0aLZu3dqoBqR4rs1PNyHEXCBPSvlDK4e9ADwmXSsxtkhK+YaUcryUcrxO57uObiklW9O3Mm3gtFaPq64+xdGjd7Jr1xCys/9BTMxSLrroBCNHriA29k6Cg2dgNJ67cGRrtBrBfy8Yw/0zh7LqhwzufvcHqmtb/bUpSr83depU1q1bR2hoKFqtltDQUEpKStixYweTJ0/mxx9/JCvLtdCw0+lk//79JCQkUFFRQXa26zpcdrudzz//nBEjRgDUPw+wdu3a+kmQGRkZVFe71lorLi7mu+++Y/jw7hvC3dd48kk/FZgvhJgDmIAgIcR7UspbGhwzHvjA/WEbDswRQtillJ94vcRecLzoOHmVeS32x1RVHSc9/b/JyXkXIXTExt7DgAGPem1hSiEEv7liGOFmI7//9CA3v7mTt26boK6mqSgtSEpKoqCggJtuuqnRcxUVFYSHh7Nnzx7uvPNOampca49NnDiRZcuWUVpayvz586mpqcHhcHDZZZfVD0d+8cUXWbt2LTqdjtDQUFasWAG4Zt0/9NBD9dcuevjhh0lK8v4l3/uLdi0rI4S4BHi4pdFl7mNWAOvaGl3my2Vl3vrpLW5fezuHf3WYkRFnl3CorDxCevrT5OauRKMxuMPlkS4dAfb5gWwe+GAvA8P8+efSicQG+3XZeylKR3hjqX/FM2pZmQaaLBvdh5Wc9AAAGpxJREFUq2xN30q4fzgjwl3V5srKQ6Sl/Zm8vH+j0fgxYMCDxMc/hNEY3eVlmZMUQ4i/gbv+uYfrX93OP5dOZGhUs11fiqIovU6/XCBzyItDSIpK4t2r/4u0tD+Tn/8RWm0gcXHLiI9/EIMhotvLdDirjFvf/p5au5O3bhvPhQnev3SzonSEqsl0n75Yk+l3w5qyy7PR2E7y8+gT7NkzjqKiL0lIeJJJk1IZPPgvPgkYgFGxQXz8yymE+Ou5+c1d/Odwrk/KoSiK4k39LmS+O/Qwb1wIISKNQYP+i0mTUklM/BN6fVjbL+5iA0L9+eiXUxgWZebu937gwz1nfF0kRVGUTulXIZOdvYJw60r2lpmZOjmdQYP+0ONWPA4PNLLyzklMOS+MRz/azyubT+CrJk1FUZTO6jchk5//MUeP3s7uIjBEPeXTy9m2JdCo4x+3TmD+2Fj+uuEof/zsME6nChpFUXqffhEyRUVfcvjwYvLsYTx7PIBbxi31dZHaZNBpeGHROJZOTWTF9lTu//deau1qoT6lf3r66acZPXo0ycnJjBs3jl27dnHJJZewZ88eAAYNGsTFFzee9zZu3DjGjBkDwObNm5k799yZF/+/vTsPr6q+Ez/+/tx7s5KEJCRoIEgE2QVSFqtUJWNZhIpWiyKlqFh0rNCCHbXO09Efoj5WcRs7iEXLDLWMIlqr4FI0Ds7TUSpLE5awayBAIA1JgKz3Jvn+/jgn4SZkJbm5Sz6v57nPPfec7znnky+XfPI9y+dkZGQwZMgQ0tPTSU9Pr7+r/xe/+AVPPPFEg/0vWLAAgOzsbK666ipGjhzJjBkzOHPmDABut5t58+YxcuRIRo8ezaZNm+rXX7t2LaNGjWLEiBE8/PDDndcxQSDk68ufPv0lu3bdTETUYH725QFuHzWfuIgma3wGHIdDePSGYSTHRvDMJ3spLnPz6tyxxESE/D+bUvW++uorNmzYwPbt24mIiKCwsBC3+/yneJ49e5a8vDz69evHnj172rz9NWvWMG7cuAbznnzySdLT05kzx3pC7euvv87f//53AObPn89zzz3HxIkTWbVqFcuWLeOJJ57gtddeA2Dnzp0UFBQwbdo0tmzZQnFxMQ899BDbtm0jOTmZO++8k8zMzAYlb0JZSI9kSkuz2bFjOhERffnSfTNFbjf3j7/f32G1i4jws4yBLJs5iq++OcXslZspLK3yd1hKdZn8/HySkpKIiIgAICkpqcmqyLfddhtr164F4M0332T27NkXvM+4uDieeuopFi5cyIIFC1i6dCnx8dYh9n379nHttdcCMHnyZN59910AcnJy6hNH7969iY+PZ+vWrXzzzTcMHjyY5GTrytVJkybVr9MdhOyfxOXl+8nOnoLLFcflI//C7a9dx8T+ExnRe4S/Q7sgt47rR6+YcO5fs52ZK77k2ZmjueJSvZdGda0DBxZTWprVqduMiUln0KCXml0+ZcoUli5dyuDBg5k0aRKzZs1i4sSJ57WbOXMmd911Fw8++CDr169nzZo1vPHGG63uf86cOURFWZU2Jk+ezLJlywCYPXs2L7/8Mk6nk7lzzz136vLLL+eDDz7gpptuYt26deTlWVeBjh49mvfff5/bb7+dvLw8tm3bRl5eHtdddx179+4lNzeX1NRU/vznPzc5EgtVITmSqaw8Qnb2JMAwevSnbDqaQ25JLgvGL/B3aB1y3dCLWDP/Sio9tdz2u6+Yv3orBws699keSgWamJgYtm3bxsqVK0lOTmbWrFn1dca8JSYmkpCQwFtvvcWwYcOIjo5u0/bXrFlDVlYWWVlZ9QkGrEKZJ06c4Pjx45SWnvt/tmrVKpYvX87YsWM5e/ZsfUX5u+++m9TUVMaNG8fixYuZMGECLpeLhIQEVqxYwaxZs7jmmmtIS0vDnwWCu1rI/aRu90mysydTXX2G9PRNREcPYfmWB+gT24cfDv2hv8PrsLH9E/ifBzNY9X/fsmLTIaa+9L/MGt+PxZMG0Ts20t/hqRDX0ojDl5xOJxkZGWRkZDBy5EhWr17dZLtZs2axYMGCJpNQey1atIglS5awZ88eHn/88foENHToUDZu3AjA/v37+fDDDwFwuVy8+OKL9etPmDCBQYOsRyPMmDGDGTNmALBy5UqcTmeH42uJiFwP/DvgBF43xvym0XKxl08HyoG7jDHb27Jue4XUSMbjKSE7eypVVUcZNepDYmPTOVR0iE8OfsK9Y+4lzBnm7xA7RVS4kwX/dBlfPJTB3Cv78/aWPDKWbeKlz/ZTVlXt7/CU6lT79u3jwIED9Z+zsrLo379/k21vvvlmHn74YaZOndqhfX788ccUFBRwxx138Oijj/Lee++Rk5MDQEFBAWA9UuDJJ5+sr+pcXl5OXamsTz/9FJfLxfDhwxusU1xczCuvvML8+fM7FF9L7KcULwemAcOB2SIyvFGzacAg+3UvsKId67ZLyIxkamrK2LnzB5SX5zBy5AZ69vweACu2rsDpcHLP2Hv8HGHn6xUTwZIbR3DXhDSW/WUfL312gD9uPsIDkwcxa1w/fRiaCgmlpaX8/Oc/p6SkBJfLxWWXXcbKlSubfIhYbGxskw8fA8jMzCQ19dxDCtetWwc0PCeTlJTEhg0bWLx4Me+88w4iQo8ePXj22WdZuHAhn3/+OW+++SbLly8H4JZbbmHevHmAlUimTp2Kw+Ggb9++Dc4HLVq0iOzsbAAee+wxBg8e3Ak906wrgIPGmG8AROQt4CYgx6vNTcAfjHWn92YRiReRFCCtDeu2S0gUyKytrWLnzhkUF2cyYsTbJCf/CIByTzmpL6QyacAk3r717U7ZVyDbfqSYpz/aw5bcYgYm9+BX1w9l8vCL9BHPqkO0QGbXaWOBTDew02vWSmPMSq/lM4HrjTHz7c9zge8aYxZ6tdkA/MYY81f7cybwK6wk0+K67RX0I5na2mpycn5McfGnDBmyqj7BAKzdtZbiyuKgP+HfVmMuSeDtf76KT3NO8ptP9nLvG9u4Ii2Rf50+lO9cEljlc5RSF6zaGDOuheVN/VXZeDTRXJu2rNsuQX08xZha9u+/h8LCP3HZZS+RkjLPa5lh+ZbljEgewbX9r/VjlF1LRJgy4mI2Lr6Wp26+nG8Ky7j5lS9ZsGY7h0/559EKSqkudRTwfoxvKnC8jW3asm67BG2SMcZw8OADnDjxX6SlPU5q6qIGy78+9jXb8rexYPyCbnm4yOV0MOe7/dn0UAaLvj+Iz/cWMOmFL1jywW6KyrrPNfqqc2iRVt/rxD7eAgwSkUtFJBy4HfigUZsPgDvEciVw2hiT38Z12yVoD5fl5i7h2LGXSU19gP79Hz1v+fIty4kNj+Uno37ih+gCR0yEiwcmD2bOdy/hpcwDvLH5MO9sO8oPRqZww+gUrhrQSy8QUC2KjIzk1KlT9OrVq1v+wdYVjDGcOnWKyMiO34ZgjKkWkYXAX7AuQ15ljNnd6GnGH2FdvnwQ6xLmeS2t25F4gvLEf17eCxw69C9cfPHdDBny+nlf/MLyQvq+0Jd7xtzDf0z/j84IN2QcLDjLK5sOsXH3SUqrqunVI5zpI1OYMboP4/on4HDoLxHVkMfj4ejRo1RWVvo7lJAWGRlJamoqYWEt32oRbE/GDLokU1DwNjk5s0hOnsnw4W9hXdbd0DN/fYZHMh9h9/27GZ7coUu8Q1alp4ZN+wpYn51P5t6TVHpquTgukhtGWQlnVGpP/atVqQAUsknGvklnK3DMGHNDo2VzsC5/AygFfmaMyW5pexeaZNzuf3DkyNMMGPA0DkfEectramsY+PJABiQM4PM7P2/39rujsqpqPttzkvXZ+XyxvwBPjeGSxGhmjLYSzpCLYjXhKBUgQjnJ/BIYB8Q1kWQmAHuMMcUiMg1YYoz5bkvb68z7ZLyt37eeG9+6kXW3rmPm8PNv1lItO13u4S85J1iffZwvD52iptYwqHcMM0b34YZRKQxIjvF3iEp1ayGZZEQkFVgNPAX8snGSadQ2AdhljOnb0jZ9lWSmrZnGjpM7yF2UGzJlZPylsLSKj3dZCWdLbhHGwIg+cdwwqg/XDEpiWEocTj2Ho1SXCtUk8w7wNBALPNhKknkQGFp3x2hzfJFkDhYdZNBvB/F4xuM8NvGxTt12d3fidCUf7sxnffZxsvJKAOvKtTH9E7giLYHxaYmM7hdPZJhvC/8p1d0FW5Jp9RJmEbkBKDDGbBORjFba/hPwU+DqZpbfi1WMrb48dmdasWUFLoeLe8aEXp0yf7u4ZyQ/vfpSfnr1pZw4Xcnfvj3FltwitnxbzHMb9wMQ7nQwul9PxqclMv7SRMb2TyAuUkeTSnVnrY5kRORpYC5QDUQCccCfjDE/adRuFPAeMM0Ys7+1HXf2SKbcU07fF/oyZeAU1s5c22nbVa0rLnOz9XAxW3KL+PrbInYdO011rcEhMCwljvFpiVxxaSLj0xJJjj3/Yg2lVNsF20imXZcw2yOZ8w6XicglwOfAHcaYL9uyrc5OMr/f/nvmr5/PF3d90a3KyASicnc1WUdK+Dq3iC25RWw/XEKFpwaAS5N6MD4tgZF9ezIsJY6hKXHERATtPcFKdbluk2S87x4VkdeBHwGH7aatFXDr1CRjjGHsyrF4aj3suG+HXm4bYDw1tew6dtoe6RSz7XARxeWe+uWXJEYzPCWOYSlxDEuJZVhKHKkJUfrvqFQTQjrJdKbOTDKbj27mqt9fxYofrOC+cfd1yjaV7xhjOHGmkj35Z8g5foY9+WfZk3+Gb0+VUfd1jIt0MTQlzk4+sQxP6cmgi2L0wgLV7WmSaaPOTDJz35vL+3vf59gvjxEbEdsp21Rdr9xdzd4TVsKxXtZ0uds61OZ0CAOSejAsJY60XtGkJkbTLyGafolRpPSM0supVbegSaaNOivJFJQV0O/Fftw75l5+O/23nRCZCiS1tYYjReX1iScn/wx7T5zleEkFtV5fXZdD6BMfRb/EKDvxRJOaEEU/OxElxYTr4TcVEoItyQT9GddVf1+Fu8bN/ePv93coygccDiEtqQdpST2YNjKlfr6nppbjJRXkFVWQV1xOXlE5ecUV5BWV89mekxSWNnycQVSY0yvpRNE3wRr99ImPok98JL1jI3UkpJQPBPVIpq5O2cDEgWTekdlJkalQUO6u5qiddLwTUF5xBUeLyjlbVd2gvcshXBQXSZ/4SDvxRNGnpzWd0jOKvvFRxEW5dDSk/E5HMl3owwMfcvj0YZ6f8ry/Q1EBJjrcxeCLYhl8UdPn6M5UesgvqeR4SQXHT1dwvKSC/JJKjpVUsP1IMR/tzMdT0/APsB7hTlLio0jpaY18kmLCSYqJICk2nF49IuqnE6PD9Rk9StmCeiQz9Y9T2V2wm9zFubgcQZ0vVYCprTUUllZxrKSC/NN2MvJKSoVnqygsdeOuqT1vXRFIiA4nKcZOPrER5xKS/R4fHUZcZBg9o8KIiwrTq+ZUm+lIposcOHWAjYc2sjRjqSYY1ekcDqF3XCS94yL5TjNtjDGcraquTzinSqsoLK3iH17ThaVudh4tobDUTWmjQ3Tewl0OO+m4iIuyk099EnI1+hxGbKSLmAgXMZEuYiPCiAxz6KE8FZCC9rfziq0rCHOEcc9YrVOm/ENEiIu0fvkPSG69faWnhn+creJUmZvTFR5OV3g4U+HhTGXddHX956IyN7mFZdb8ympqals+4uAQq2BpbGRYffI5l4RcDefZ0z0irGU9Is7N7xHhItylh/pU5wnKJFPuKec/s/6THw3/ERfHXOzvcJRqk8gwp3V1W2J0u9YzxlDmruGMV2IqraqmtKqas5XWe2ml92dreUm5m7zi8vpldfcbtSbc6bCTkJMe4S5i7YTknZSiw51EhTuJDrPeo8JdRIU5iQ53Emm/132Osqf1PFX3FJRJ5r93/jcllSXcP04vW1ahT0TqRxp94qMueDs1taY+OZXZCamsbrrq3HRpVQ2lVR7KqmrqE1hRmZsjRVbCKquqptxTQ3tP54Y7HUSGOYgOdxHVKBlF1iWrMAdRYeeSVlS4o365tZ6DyDAnES4nES5re3XTES4nEWEOwp0OHHo5esAIuhP/xhjGrBxDTW0N2fdl63FopfzAGENVdS0V7hrKPTVUuO2Xp4Zyd7XXdE2D6cq65R5r3UpPw2V1bSs8Nbirz7+ooq3CnOKVfBxEhFnT4XWfXc6GCarB9PnrRXrPczkJd1nJLNzlIMwpjT5b7y6H+OT3k57497HNRzeTdSKLV3/wqiYYpfxERIi0RxgJPtpHdU0tldUNk1GF20pIVdU1VFXXUlVdi7u61vrsqbXn2cs81rS7uuH8ukRXXF5b/9lqf26bnUEEK+E0SkZhTgc/vuIS5l8zoFP2E+iCLskYDFMHTmXOqDn+DkUp5UMup4MYp6PLHwVhjMFdU0ulp+nkVWmPsjw1Bk+NleTc9run0bvbq4132+70XKWgO1ymlFLdWbAdLtPLPZRSSvmMJhmllOomRCRRRD4VkQP2e5On1ETkehHZJyIHReQRr/lLROSYiGTZr+mt7VOTjFJKdR+PAJnGmEFApv25ARFxAsuBacBwYLaIDPdq8qIxJt1+fdTaDjXJKKVU93ETsNqeXg38sIk2VwAHjTHfGGPcwFv2ehdEk4xSSgUXl4hs9Xrd2451LzLG5APY772baNMXyPP6fNSeV2ehiOwQkVXNHW5rEGw7glNKKeV/1caYcc0tFJHPgKbqbf26jdtv6gbEusuQVwBP2J+fAJ4H7m5pY5pklFIqhBhjJjW3TEROikiKMSZfRFKAgiaaHQX6eX1OBY7b2z7pta3XgA2txaOHy5RSqvv4ALjTnr4TeL+JNluAQSJyqYiEA7fb62Enpjo3A7ta26HfbsYUkVqg4gJXdwHNP5zD/wI9Pgj8GDW+jtH4OiaQ44syxlzQAEFEegFvA5cAR4BbjTFFItIHeN0YM91uNx14CXACq4wxT9nz3wDSsQ6X5QL/XHeOp9l9+ivJdISIbG3pmKS/BXp8EPgxanwdo/F1TKDHF0z0cJlSSimf0SSjlFLKZ4I1yaz0dwCtCPT4IPBj1Pg6RuPrmECPL2gE5TkZpZRSwSFYRzJKKaWCgCYZpZRSPhPQSaa5ctNey0VEXraX7xCRMV0YWz8R+R8R2SMiu0VkURNtMkTktFdZ7Me6Kj57/7kistPe99Ymlvuz/4Z49UuWiJwRkcWN2nR5/9n1mApEZJfXvA6VR++C+JaJyF773/A9EYlvZt0Wvw8+jK9N5eH92H9rvWLLFZGsZtb1ef+FJGNMQL6wbgI6BAwAwoFsYHijNtOBj7Fq7VwJ/K0L40sBxtjTscD+JuLLADb4sQ9zgaQWlvut/5r4tz4B9Pd3/wHXAmOAXV7zngUesacfAZ5p5mdo8fvqw/imAC57+pmm4mvL98GH8S0BHmzDd8Av/ddo+fPAY/7qv1B8BfJIpi3lpm8C/mAsm4H4RmUPfMYYk2+M2W5PnwX20LBSaTDwW/818n3gkDHmsB/23YAx5n+Bokazu7w8enviM8ZsNMbU3Z2+GavWlF80039t4bf+qyMiAtwGvNnZ++3OAjnJtFZuuq1tfE5E0oDvAH9rYvFVIpItIh+LyIguDcwq/bBRRLY1Uw48IPoPqzZSc/+x/dl/dTqjPHpXuRtrdNqU1r4PvtRaefhA6L9rgJPGmAPNLPdn/wWtQE4yLZWbbk8bnxKRGOBdYLEx5kyjxduxDgGNBn4L/LkrYwO+Z4wZg/WEuwUicm2j5YHQf+HAjcC6Jhb7u//aIxD68tdY9bbWNNOkte+Dr6wABmLVvMrHOiTVmN/7D5hNy6MYf/VfUAvkJNNsuel2tvEZEQnDSjBrjDF/arzcGHPGGFNqT38EhIlIUlfFZ4ypK89dALyHdUjCm1/7zzYN2G68SojX8Xf/eTlZdxhRLqA8elcQkTuBG4A5xj6B0Fgbvg8+YYw5aYypMcbUAq81s19/958LuAVY21wbf/VfsAvkJNNsuWkvHwB32FdJXQmcNq1UBO0s9vHb3wN7jDEvNNPmYrsdInIFVn+f6qL4eohIbN001snhxmW5/dZ/Xpr969Gf/ddIh8qj+5qIXA/8CrjRGFPeTJu2fB98FV9bysP7rf9sk4C9xpijTS30Z/8FPX9fedDSC+vqp/1YV5382p53H3CfPS3Acnv5TmBcF8Z2NdZwfgeQZb+mN4pvIbAb60qZzcCELoxvgL3fbDuGgOo/e//RWEmjp9c8v/YfVsLLBzxYf13/FOgFZAIH7PdEu20f4KOWvq9dFN9BrPMZdd/DVxvH19z3oYvie8P+fu3AShwpgdR/9vz/qvveebXt8v4LxZeWlVFKKeUzgXy4TCmlVJDTJKOUUspnNMkopZTyGU0ySimlfEaTjFJKKZ/RJKNUG9lVoTf4Ow6lgokmGaWUUj6jSUaFHBH5iYh8bT/343ci4hSRUhF5XkS2i0imiCTbbdNFZLPXs1gS7PmXichndnHO7SIy0N58jIi8Yz+/ZY1XRYLfiEiOvZ3n/PSjKxVwNMmokCIiw4BZWMUM04EaYA7QA6tG2hjgC+D/2av8AfiVMWYU1l3pdfPXAMuNVZxzAtZd4mBV214MDMe6C/x7IpKIVS5lhL2dJ337UyoVPDTJqFDzfWAssMV+wuH3sZJBLeeKH/4RuFpEegLxxpgv7PmrgWvtGlV9jTHvARhjKs25mmBfG2OOGqvYYxaQBpwBKoHXReQWoMn6YUp1R5pkVKgRYLUxJt1+DTHGLGmiXUv1lJoqO1+nymu6BuuJlNVYFXnfxXqg2SftjFmpkKVJRoWaTGCmiPQGEJFEEemP9V2fabf5MfBXY8xpoFhErrHnzwW+MNZzgY6KyA/tbUSISHRzO7SfKdTTWI8jWIz13BSlFODydwBKdSZjTI6I/BvWEwwdWNV2FwBlwAgR2QacxjpvA1bp/lftJPINMM+ePxf4nYgstbdxawu7jQXeF5FIrFHQA538YykVtLQKs+oWRKTUGBPj7ziU6m70cJlSSimf0ZGMUkopn9GRjFJKKZ/RJKOUUspnNMkopZTyGU0ySimlfEaTjFJKKZ/5/9Je3bjCSPAVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax1 = plt.subplots()\n",
    "\n",
    "ax1.plot(val_losses, label='val loss')\n",
    "\n",
    "ax2 = ax1.twinx()\n",
    "ax2.plot(men, label='MEN', c='g')\n",
    "ax2.plot(ws353, label='WS353', c='m')\n",
    "ax2.plot(simlex999, label='SIMLEX999', c='y')\n",
    "\n",
    "ax1.legend(loc=[0.07, 0.9])\n",
    "ax2.legend(loc=[0.7, 0.15])\n",
    "\n",
    "ax1.set_xlabel('epochs');"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}