Reward_Optimizing_Slate_Recommendation_with_DL_and_MIPS_Part_2.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fd2976f6-fa07-413c-a593-ded6265c4dee",
        "outputId": "2b31385c-136f-47ef-9ad9-df2dbfa7bf8f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gdown in /usr/local/lib/python3.7/dist-packages (4.4.0)\n",
            "Collecting faiss-cpu\n",
            "  Downloading faiss_cpu-1.7.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 8.6 MB 4.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gdown) (1.15.0)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.7/dist-packages (from gdown) (2.23.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from gdown) (4.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from gdown) (4.64.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from gdown) (3.7.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (2022.6.15)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (1.7.1)\n",
            "Installing collected packages: faiss-cpu\n",
            "Successfully installed faiss-cpu-1.7.2\n",
            "Downloading...\n",
            "From: https://drive.google.com/u/0/uc?id=1nMQibdolybOL1GWKB1OYxqU9KtEnrhB1\n",
            "To: /content/setup.zip\n",
            "100% 72.4M/72.4M [00:02<00:00, 35.8MB/s]\n",
            "Archive:  setup.zip\n",
            "   creating: setup/\n",
            "  inflating: setup/.DS_Store         \n",
            "  inflating: __MACOSX/setup/._.DS_Store  \n",
            "   creating: setup/twitch/\n",
            "  inflating: setup/twitch/.DS_Store  \n",
            "  inflating: __MACOSX/setup/twitch/._.DS_Store  \n",
            "   creating: setup/twitch/N:100000_K:10_seed:0/\n",
            "   creating: setup/twitch/.ipynb_checkpoints/\n",
            "   creating: setup/twitch/N:100000_K:10_seed:0/data/\n",
            "   creating: setup/twitch/N:100000_K:10_seed:0/data/test/\n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/prod_emb.npy  \n",
            "   creating: setup/twitch/N:100000_K:10_seed:0/data/train/\n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/index  \n",
            "   creating: setup/twitch/N:100000_K:10_seed:0/data/val/\n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/test/rewards.npz  \n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/test/contexts.pt  \n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/train/rewards.npz  \n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/train/contexts.pt  \n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/val/rewards.npz  \n",
            "  inflating: setup/twitch/N:100000_K:10_seed:0/data/val/contexts.pt  \n"
          ]
        }
      ],
      "source": [
        "!pip install gdown faiss-cpu\n",
        "!gdown https://drive.google.com/u/0/uc?id=1nMQibdolybOL1GWKB1OYxqU9KtEnrhB1&export=download\n",
        "!unzip setup"
      ],
      "id": "fd2976f6-fa07-413c-a593-ded6265c4dee"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4e01124b-b8c9-45cc-9e6e-fe65f384b50c",
        "outputId": "39ea1fee-53b5-40d4-90af-5779395db62b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading the different splits\n",
            "Loading the embeddings\n",
            "number of products is 790100\n",
            "Loading the index\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from scipy import sparse\n",
        "import os\n",
        "\n",
        "import scipy.sparse.linalg as sp_l\n",
        "import faiss\n",
        "import torch \n",
        "from torch import nn, optim\n",
        "from tqdm import tqdm\n",
        "import time\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "sns.set()\n",
        "\n",
        "def load_tensor_spmx_dataset(path, split = 'train'):\n",
        "    load_dir = os.path.join(path, split)\n",
        "    contexts = torch.load(os.path.join(load_dir, 'contexts.pt'))\n",
        "    rewards = sparse.load_npz(os.path.join(load_dir, 'rewards.npz'))\n",
        "    return contexts, rewards\n",
        "\n",
        "seed = 0\n",
        "np.random.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "\n",
        "print('Loading the different splits')\n",
        "exp_folder_path = 'setup/twitch/N:100000_K:10_seed:0'\n",
        "data_folder = os.path.join(exp_folder_path, 'data')\n",
        "\n",
        "contexts_train, rewards_train = load_tensor_spmx_dataset(data_folder, split = 'train')\n",
        "contexts_val, rewards_val = load_tensor_spmx_dataset(data_folder, split = 'val')\n",
        "contexts_test, rewards_test = load_tensor_spmx_dataset(data_folder, split = 'test')\n",
        "\n",
        "print('Loading the embeddings')\n",
        "prod_emb = np.load(os.path.join(data_folder, 'prod_emb.npy'))\n",
        "P = prod_emb.shape[0]\n",
        "print('number of products is %d'%P)\n",
        "\n",
        "print('Loading the index')\n",
        "index = faiss.read_index(os.path.join(data_folder, 'index'))"
      ],
      "id": "4e01124b-b8c9-45cc-9e6e-fe65f384b50c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e6df754c-bc5f-4c72-a299-215de4f418a5"
      },
      "outputs": [],
      "source": [
        "class policy_model(torch.nn.Module):\n",
        "    def __init__(self, emb):\n",
        "        super(policy_model, self).__init__()\n",
        "        self.emb = torch.Tensor(emb.T)\n",
        "        self.K, _ = self.emb.shape\n",
        "        self.theta = torch.nn.Parameter(0.005 * torch.randn(self.K, self.K))\n",
        "        self.log_sigma = torch.nn.Parameter(torch.zeros(1))\n",
        "\n",
        "    def forward(self, x):\n",
        "        x_transformed = torch.matmul(x, self.theta)\n",
        "        log_unnormalized = torch.matmul(x_transformed, self.emb)\n",
        "        return nn.functional.softmax(log_unnormalized, dim = -1), log_unnormalized\n",
        "    \n",
        "    def sample(self, x, n_samples = 1):\n",
        "        x_transformed = torch.matmul(x, self.theta)\n",
        "        log_unnormalized = torch.matmul(x_transformed, self.emb)\n",
        "        scores = torch.exp(log_unnormalized - log_unnormalized.max())\n",
        "        actions = torch.multinomial(scores, n_samples, replacement=True)\n",
        "        return actions\n",
        "    \n",
        "    def argmax(self, x):\n",
        "        x_transformed = torch.matmul(x, self.theta)\n",
        "        log_unnormalized = torch.matmul(x_transformed, self.emb)\n",
        "        return torch.argmax(log_unnormalized, dim = 1, keepdim = True)\n",
        "    \n",
        "    def Xtransformed(self, x):\n",
        "        x_transformed = torch.matmul(x, self.theta)\n",
        "        return x_transformed"
      ],
      "id": "e6df754c-bc5f-4c72-a299-215de4f418a5"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "41773a99-77f4-4524-bd88-98fb2a336f81"
      },
      "source": [
        "### Training/Testing Routine"
      ],
      "id": "41773a99-77f4-4524-bd88-98fb2a336f81"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bf755570-3241-4d50-a679-cb10c544826f"
      },
      "source": [
        "We define the code to test out the policies once trained."
      ],
      "id": "bf755570-3241-4d50-a679-cb10c544826f"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "95abc358-88c1-432f-9b4e-3168eed3b2d8"
      },
      "outputs": [],
      "source": [
        "def test_policy(policy, index, contexts, rewards, bsize):\n",
        "    \n",
        "    greed_reward_index, greed_reward = 0., 0.\n",
        "    N_test = contexts.size(0)\n",
        "    idxlist = np.arange(N_test)\n",
        "    \n",
        "    for bnum, st_idx in tqdm(enumerate(range(0, N_test, bsize))):\n",
        "            end_idx = min(st_idx + bsize, N_test)\n",
        "            indices = idxlist[st_idx:end_idx]\n",
        "            len_indices = len(indices)\n",
        "            help_broadcast = np.arange(len_indices)\n",
        "            \n",
        "            r = rewards[indices]\n",
        "            c = contexts[indices]\n",
        "            \n",
        "            _, top_actions_index = index.search(policy.Xtransformed(c).cpu().detach().numpy(), k = 1)\n",
        "            top_rewards_index = rewards[indices[:,np.newaxis], top_actions_index].A\n",
        "            \n",
        "            top_actions = policy.argmax(c).cpu().numpy()\n",
        "            top_rewards = rewards[indices].A[help_broadcast[:,np.newaxis], top_actions]\n",
        "            \n",
        "            \n",
        "            r_a_index = top_rewards_index.sum()\n",
        "            r_a = top_rewards.sum()\n",
        "\n",
        "            greed_reward_index += r_a_index/N_test\n",
        "            greed_reward += r_a/N_test\n",
        "                \n",
        "    return greed_reward_index, greed_reward"
      ],
      "id": "95abc358-88c1-432f-9b4e-3168eed3b2d8"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d803048c-66fe-4d1b-b0cd-37302521cf84"
      },
      "source": [
        "We also define the training routine boiler-plate that every algorithm will use."
      ],
      "id": "d803048c-66fe-4d1b-b0cd-37302521cf84"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2314f64b-e4e6-4faf-86bf-6bd8da1ab67e"
      },
      "outputs": [],
      "source": [
        "def training_routine(algorithm_loss, pi, index, n_samples, P, trunc_at, eps, contexts, rewards, epochs, bsize, lr, reg, val_contexts, val_rewards) :\n",
        "    \n",
        "    optimizer = torch.optim.Adam(pi.parameters(), lr=lr, weight_decay=reg)\n",
        "    N_train = contexts.size(0)\n",
        "    idxlist = np.arange(N_train)\n",
        "    proposal_probs = torch.ones(bsize, P) * eps/P\n",
        "    \n",
        "    train_m_rewards = []\n",
        "    val_max, val_max_index = [0.]*epochs, [0.]*epochs\n",
        "    \n",
        "    true_duration = 0.\n",
        "    \n",
        "    for i in range(epochs):\n",
        "        print(\"epoch number %d\"%i)\n",
        "        np.random.shuffle(idxlist)\n",
        "        # train for one epoch\n",
        "        for bnum, st_idx in tqdm(enumerate(range(0, N_train, bsize))):\n",
        "            base_time = time.time()\n",
        "            \n",
        "            end_idx = min(st_idx + bsize, N_train)\n",
        "            indices = idxlist[st_idx:end_idx]\n",
        "            indices_len = len(indices)\n",
        "            \n",
        "            X = contexts[indices]\n",
        "\n",
        "            loss = algorithm_loss(pi, contexts, rewards, indices, index, n_samples, proposal_probs)\n",
        "            \n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            optimizer.zero_grad()\n",
        "            \n",
        "            true_duration += time.time() - base_time\n",
        "            \n",
        "            _, top_actions = index.search(pi.Xtransformed(X).detach().numpy(), k = 1)\n",
        "            top_rewards = rewards[indices[:,np.newaxis], top_actions].A\n",
        "            r_a = top_rewards.mean()\n",
        "            train_m_rewards.append(r_a)\n",
        "            \n",
        "        print('Computing metrics on val')\n",
        "        max_perf_index, max_perf = test_policy(pi, index, val_contexts, val_rewards, 512)\n",
        "        val_max_index[i], val_max[i] = max_perf_index, max_perf\n",
        "        print('Argmax reward on the validation : INDEX %.4f, TRUE %.4f'%(max_perf_index, max_perf))\n",
        "        \n",
        "    return train_m_rewards, val_max_index, val_max, true_duration\n"
      ],
      "id": "2314f64b-e4e6-4faf-86bf-6bd8da1ab67e"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e12b2ef9-da7a-4bf7-84ac-bdd953e7f485"
      },
      "source": [
        "## Reinforce"
      ],
      "id": "e12b2ef9-da7a-4bf7-84ac-bdd953e7f485"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c4e808c8-0b8c-47c2-8f20-67c24b4d2bd6"
      },
      "source": [
        "In large scale recommendation problems, we usually deal with a considerable amount of observations making stochastic gradient descent and its variants suitable for such application. For a single observation $x_i$, we are interested in the gradient of $\\hat{R}_i(\\pi_\\theta) = \\mathbf{\\mathbb{E}}_{a \\sim \\pi_\\theta(.|x_i)}[\\hat{r}(a, x_i)]$ which can be derived using the log trick giving the following formula:\n",
        "\n",
        "\\begin{align}\n",
        "\\label{reinforce}\n",
        "\\nabla_\\theta \\hat{R}_i(\\pi_\\theta) = \\mathbf{\\mathbb{E}}_{a \\sim \\pi_\\theta(.|x_i)}[\\hat{r}(a, x_i)\\nabla_\\theta \\log \\pi_\\theta(a|x_i)].\n",
        "\\end{align}\n"
      ],
      "id": "c4e808c8-0b8c-47c2-8f20-67c24b4d2bd6"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "41fdd430-0107-43d7-94de-fef30b0c67e1"
      },
      "outputs": [],
      "source": [
        "def reinforce_loss(pi, contexts, rewards, indices, index, n_samples, proposal_probs):\n",
        "    \n",
        "    X = contexts[indices]\n",
        "    indices_len = len(indices)\n",
        "    probs, _ = pi(X)\n",
        "    with torch.no_grad():\n",
        "        actions = torch.multinomial(probs, n_samples, replacement=True)\n",
        "        r_a = torch.Tensor(rewards[indices[:,np.newaxis], actions].A)\n",
        "\n",
        "    help_broadcast = np.arange(indices_len)[:,np.newaxis]\n",
        "    return -torch.mean(r_a * torch.log(probs[help_broadcast, actions]))"
      ],
      "id": "41fdd430-0107-43d7-94de-fef30b0c67e1"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a4d269e8-b1f8-47f4-9232-eacfb0e1d0c7"
      },
      "source": [
        "This gradient can be computed exactly as an expectation over a discrete distribution, our current policy $\\pi_\\theta$, when the action space is of small size. Once the size of the catalog $P$ is in the order of millions, an exact gradient update becomes a bottleneck for the optimization process because of the complexity of the following computations:\n",
        "\n",
        "**1 - Computing $\\nabla_\\theta \\log \\pi_\\theta(.|x_i)$**: \n",
        "\n",
        "We need to deal with the normalizing constant $Z_\\theta(x_i)$ present in the computation of $\\nabla_\\theta \\log \\pi_\\theta(.|x_i)$. Indeed, $Z_\\theta(x_i)$ is a sum over all the action space and its computation needs to be avoided if we hope to reduce the complexity of the gradient update.\n",
        "\n",
        "**2 - Computing the expectation**:\n",
        "\n",
        "The expectation is a sum over all the action space and is obviously computed in $\\mathcal{O}(P)$. To avoid this expensive sum, we can resort to sampling from $\\pi_\\theta$ to approximate the gradient. This allows us to obtain the Reinforce estimator, an unbiased estimator of the expectation but does not change the complexity of the method which stays linear in the catalog size. Indeed, sampling needs the computation of $Z_\\theta(x_i)$ or can be done with the gumbel trick which both scale in $\\mathcal{O}(P)$. To lower the time complexity, we need to avoid sampling directly from $\\pi_\\theta$ and use Monte Carlo techniques instead such as importance sampling/rejection sampling with carefully chosen proposals to achieve fast sampling and accurate gradient approximation.\n"
      ],
      "id": "a4d269e8-b1f8-47f4-9232-eacfb0e1d0c7"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e94506ac-8026-4057-8d58-94af8909d524"
      },
      "source": [
        "# Accelerating Reinforce"
      ],
      "id": "e94506ac-8026-4057-8d58-94af8909d524"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fbd010ac-ed80-46a0-9852-012bc8984f25"
      },
      "source": [
        "As pointed out in the previous section, we need a workaround to deal with the presence of the normalizing constant in the gradient. For a fixed observations $x_i$, we can push further the computation of $\\nabla_\\theta \\log \\pi_\\theta(a|x_i)$ to obtain a quantity that does not involve $Z_\\theta(x_i)$. Indeed, we have for a fixed action $a$:\n",
        "\n",
        "\\begin{align*}\n",
        "\\nabla_\\theta \\log \\pi_\\theta(a|x_i) &= \\nabla_\\theta f_\\theta(a, x_i) - \\nabla_\\theta \\log Z_\\theta(x_i) \\\\\n",
        "&= \\nabla_\\theta f_\\theta(a, x_i) - \\frac{\\nabla_\\theta Z_\\theta(x_i)}{Z_\\theta(x_i)} \\\\\n",
        "&= \\nabla_\\theta f_\\theta(a, x_i) - \\sum_b \\pi_\\theta(b|x_i) \\nabla_\\theta f_\\theta(b, x_i) \\\\\n",
        "&= \\nabla_\\theta f_\\theta(a, x_i) -  \\mathbf{\\mathbb{E}}_{b \\sim \\pi_\\theta(.|x_i)}[\\nabla_\\theta f_\\theta(b, x_i)]\n",
        "\\end{align*}\n",
        "\n",
        "\n",
        "Injecting the above expression of $\\nabla_\\theta \\log \\pi_\\theta(a|x_i)$ into the reinforce gradient leads us to the following covariance gradient:\n",
        "\n",
        "\\begin{align}\n",
        "\\nabla_\\theta \\hat{R}_i(\\pi_\\theta) = \\mathbf{Cov}_{a \\sim \\pi_\\theta(.|x_i)}[\\hat{r}(a, x_i),  \\nabla_\\theta f_\\theta(a, x_i)] \\label{cov_grad}\n",
        "\\end{align}\n",
        "\n",
        "\n",
        "with \n",
        "\\begin{align*}\n",
        "    \\mathbf{Cov}[A, \\boldsymbol{B}] &= \\mathbf{\\mathbb{E}}[A.\\boldsymbol{B}] - \\mathbf{\\mathbb{E}}[A].\\mathbf{\\mathbb{E}}[\\boldsymbol{B}] \\\\\n",
        "    &= \\mathbf{\\mathbb{E}}[(A - \\mathbf{\\mathbb{E}}[A] ).(\\boldsymbol{B} - \\mathbf{\\mathbb{E}}[\\boldsymbol{B}])] \n",
        "\\end{align*}\n",
        "\n",
        "Now that we got rid of the normalizing constant inside the gradient, we need to approximate the expectation with Self Normalized Importance Sampling:\n",
        "\n",
        "$$\\mathbf{\\mathbb{E}}_{\\pi_\\theta(.|x_i)}[g(a)] \\approx \\sum_{s = 1}^S \\bar{\\omega}_s g(a_s)$$\n",
        " \n",
        "\n",
        "with $a_s \\sim q \\quad \\forall s$, $\\omega_s = \\frac{\\exp \\{f(a_s, x_i)\\}}{q(a_s)}$ and $\\bar{\\omega}_s = \\frac{\\omega_s}{\\sum_{s' = 1}^S \\omega_{s'}}$."
      ],
      "id": "fbd010ac-ed80-46a0-9852-012bc8984f25"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "04d06a93-1479-4fea-884a-d98f9b6ade2e"
      },
      "source": [
        "### SNIPS - Uniform Proposal"
      ],
      "id": "04d06a93-1479-4fea-884a-d98f9b6ade2e"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9ac506c8-09de-445a-a4be-a24f184aebc8"
      },
      "source": [
        "we use the uniform proposal to approximate the gradient :\n",
        "\n",
        "$$q(a|x) = \\frac{1}{P}$$ with P the catalg size."
      ],
      "id": "9ac506c8-09de-445a-a4be-a24f184aebc8"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3ca6aa17-c00e-4c43-b174-5cdda677d23e"
      },
      "outputs": [],
      "source": [
        "def uniform_loss(pi, contexts, rewards, indices, index, n_samples, proposal_probs):\n",
        "    \n",
        "    X = contexts[indices]\n",
        "    len_indices = len(indices)\n",
        "    x_transformed = torch.matmul(X, pi.theta)\n",
        "    a_samples = torch.randint(P, [len_indices, n_samples])\n",
        "    a_embs = pi.emb[:, a_samples]\n",
        "\n",
        "    log_p_tilde = torch.einsum('ij,jik->ik', x_transformed, a_embs)\n",
        "    with torch.no_grad():\n",
        "        ws = torch.nn.functional.softmax(log_p_tilde, dim=-1) # SNIPS\n",
        "\n",
        "    r_a = torch.Tensor(rewards[indices[:,np.newaxis], a_samples].A)\n",
        "\n",
        "    mean_log_p_tilde, mean_rewards = torch.sum(ws * log_p_tilde, dim=-1, keepdim=True), torch.sum(ws * r_a, dim=-1, keepdim=True)\n",
        "    \n",
        "    return - torch.mean(torch.sum(ws * (log_p_tilde - mean_log_p_tilde) * (r_a - mean_rewards), dim=-1))"
      ],
      "id": "3ca6aa17-c00e-4c43-b174-5cdda677d23e"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90b307f4-4eb2-4ede-8ecb-57bcd977e8cc"
      },
      "source": [
        "## SNIPS - Mixture Proposal"
      ],
      "id": "90b307f4-4eb2-4ede-8ecb-57bcd977e8cc"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "48a59782-a4fd-4fb4-a306-d29813109620"
      },
      "source": [
        "We use a proposal that uses the MIPS index in training time as well, which is a mixture between a uniform (to cover the first training phase) and the head of the distribution (covers the late training phases when the policy concentrates on the good items).\n",
        "\n",
        "By constructing the head of the distribution (K top actions, with K a hyperparameter) $\\alpha_K(x_i)= {\\rm argsort}(h_{\\theta}(x_i)^T\\beta)_{1:K}$ with the help of MIPS, we define :\n",
        "\n",
        "$$\n",
        "    q_{K, \\epsilon}(a|x_i)= \n",
        "\\begin{cases}\n",
        "    \\frac{\\epsilon}{P} + (1-\\epsilon)\\kappa(a|x_i),              & \\text{if } a \\in \\alpha_K(x_i) \\\\\n",
        "    \\frac{\\epsilon}{P},& \\text{otherwise }.\n",
        "\\end{cases}\n",
        "$$\n",
        "\n",
        "Where $\\epsilon$ is a parameter that controls the mixture and \n",
        "$$\n",
        "\\kappa(a|x_i) = \\frac{\\exp(h_{\\theta}(x_i)^T\\beta_a)}{\\sum_{a' \\in \\alpha_K(x_i)}  \\exp(h_{\\theta}(x_i)^T\\beta_{a'})}1[a \\in \\alpha_K(x_i)].\n",
        "$$\n",
        "\n",
        "Note that $\\epsilon = 1.$ retrieves the uniform proposal. "
      ],
      "id": "48a59782-a4fd-4fb4-a306-d29813109620"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "101ab6ff-d74c-48cf-8929-1e8b30775b22"
      },
      "outputs": [],
      "source": [
        "def mixture_loss(pi, contexts, rewards, indices, index, n_samples,proposal_probs):\n",
        "    \n",
        "    len_indices = len(indices)\n",
        "    help_broadcast = np.arange(len_indices)[:,np.newaxis]\n",
        "\n",
        "    X = contexts[indices]\n",
        "    x_transformed = torch.matmul(X, pi.theta)\n",
        "\n",
        "    query = x_transformed.cpu().detach().numpy()\n",
        "    topK_scores, topK_indices = index.search(query, k = trunc_at)\n",
        "\n",
        "\n",
        "    topK_indices, topK_scores = torch.tensor(topK_indices), torch.tensor(topK_scores)\n",
        "    topK_probs = torch.nn.functional.softmax(topK_scores, dim=-1)\n",
        "\n",
        "    proposal_probs[help_broadcast, topK_indices] += (1. - eps) * topK_probs\n",
        "\n",
        "    uni_n_samples = int(n_samples * eps)\n",
        "    a_samples_uni = torch.randint(P, [len_indices, uni_n_samples])\n",
        "    a_samples_topK = topK_indices[help_broadcast, torch.multinomial(topK_probs, n_samples - uni_n_samples, replacement=True)]\n",
        "\n",
        "    a_samples = torch.cat([a_samples_uni, a_samples_topK], dim=1)\n",
        "\n",
        "    a_embs = pi.emb[:, a_samples]\n",
        "    log_p_tilde = torch.einsum('ij,jik->ik', x_transformed, a_embs)\n",
        "\n",
        "    # SNIPS\n",
        "    with torch.no_grad():\n",
        "        log_ws_tilde = log_p_tilde - torch.log(proposal_probs[help_broadcast, a_samples])\n",
        "        ws = torch.nn.functional.softmax(log_ws_tilde, dim=-1)\n",
        "\n",
        "    r_a = torch.Tensor(rewards[indices[:,np.newaxis], a_samples].A)\n",
        "    mean_log_p_tilde, mean_rewards = torch.sum(ws * log_p_tilde, dim=-1, keepdim=True), torch.sum(ws * r_a, dim=-1, keepdim=True)\n",
        "    \n",
        "    proposal_probs[help_broadcast, topK_indices] = eps/P\n",
        "    \n",
        "    return - torch.mean(torch.sum(ws * (log_p_tilde - mean_log_p_tilde) * (r_a - mean_rewards), dim=-1))\n"
      ],
      "id": "101ab6ff-d74c-48cf-8929-1e8b30775b22"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e7a6c657-c2fe-40d5-8684-d5e6c66d23a9"
      },
      "source": [
        "## Experiments :"
      ],
      "id": "e7a6c657-c2fe-40d5-8684-d5e6c66d23a9"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ba9bd2ac-9450-4052-89f4-f659d3533555"
      },
      "outputs": [],
      "source": [
        "epochs = 1\n",
        "bsize = 32\n",
        "reg = 1e-10\n",
        "\n",
        "lr = 1e-4\n",
        "n_samples = 1000\n",
        "eps = 0.8\n",
        "trunc_at = 256"
      ],
      "id": "ba9bd2ac-9450-4052-89f4-f659d3533555"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2019e2bc-29b2-45e1-b7ef-f46a6411da33"
      },
      "source": [
        "### Training the policies"
      ],
      "id": "2019e2bc-29b2-45e1-b7ef-f46a6411da33"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "df346fce-36a4-4090-bc88-f155032cb361",
        "outputId": "6c0d4538-22be-4167-f27e-7fdb47407e49"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch number 0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2500it [11:48,  3.53it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on val\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "20it [00:39,  1.98s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Argmax reward on the validation : INDEX 0.1922, TRUE 0.2264\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "# Reinforce\n",
        "policy_reinforce = policy_model(prod_emb)\n",
        "train_r, val_r, _, duration_r = training_routine(reinforce_loss, policy_reinforce, index, n_samples, P, trunc_at, eps, \n",
        "                                                 contexts_train, rewards_train, epochs, bsize, lr, reg, contexts_val, rewards_val)"
      ],
      "id": "df346fce-36a4-4090-bc88-f155032cb361"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8b48183a-a871-4ada-85fe-ee354727d32c",
        "outputId": "5157cd02-0c41-483f-a20e-63c13057e7fc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch number 0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2500it [00:47, 52.83it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on val\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "20it [00:36,  1.84s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Argmax reward on the validation : INDEX 0.2174, TRUE 0.2266\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "# Uniform proposal\n",
        "policy_uniform = policy_model(prod_emb)\n",
        "train_u, val_u, _, duration_u = training_routine(uniform_loss, policy_uniform, index, n_samples, P, trunc_at, eps, \n",
        "                                                              contexts_train, rewards_train, epochs, bsize, lr, reg, contexts_val, rewards_val)"
      ],
      "id": "8b48183a-a871-4ada-85fe-ee354727d32c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d01ace50-fd71-4959-a9a5-2e48a7f83e6a",
        "outputId": "e74dad50-27f0-459b-be57-f116a945e5b2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch number 0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2500it [01:14, 33.48it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on val\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "20it [00:36,  1.84s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Argmax reward on the validation : INDEX 0.2246, TRUE 0.2252\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "# Mixture proposal\n",
        "policy_mixture = policy_model(prod_emb)\n",
        "train_m, _, val_m, duration_m = training_routine(mixture_loss, policy_mixture, index, n_samples, P, trunc_at, eps, \n",
        "                                                 contexts_train, rewards_train, epochs, bsize, lr, reg, contexts_val, rewards_val)"
      ],
      "id": "d01ace50-fd71-4959-a9a5-2e48a7f83e6a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9f4864bc-c8a0-4346-9dd7-ca0b5cc72e61"
      },
      "source": [
        "### Relative speed-up :"
      ],
      "id": "9f4864bc-c8a0-4346-9dd7-ca0b5cc72e61"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3b67bc2c-7192-4239-8bcd-fcf56dd8245c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 268
        },
        "outputId": "fea83d28-30fb-4cad-af49-9e60885b4c87"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD7CAYAAACCEpQdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZkUlEQVR4nO3df1RUdf7H8ReDDCboCnyRxh8nzV1cjtpaWZ5qldOoKxYExSKG9sPCUtulbD3hj7OSJHkg8qSspZZH2/VHHI8hiz/J0+7W5ra1a3vW1NJciVBAAkvQFGXm+4en2VhAB+4A4uf5+OveuZ/7+byHy7zmzmdm7vi53W63AABGsXV2AQCAjkf4A4CBCH8AMBDhDwAGIvwBwECEPwAYiPAHAAN16+wCvHXq1Bm5XHwlAQC8YbP5KSQkqMXtXSb8XS434Q8APsK0DwAYiPAHAAMR/gBgIMIfAAxE+AOAgQh/ADAQ4Q8ABuoyn/MHcHXr9aNABdrtnV3GNe98fb1Of3vecj+EPwCfCLTb9ejapzu7jGveumnLJFkPf6Z9AMBAhD8AGIjwBwADEf4AYCDCHwAMRPgDgIEIfwAwEOEPAAYi/AHAQF59wzc7O1u7d+/W8ePHVVRUpMjISJWVlempp57ytKmtrVVdXZ0++uijJvvn5eVp48aN6tOnjyTplltuUUZGho/uAgCgtbwK/7Fjx+rhhx/WlClTPLf1799fhYWFnvWsrCw1NDS02EdCQoLS09MtlAoA8BWvwn/kyJGX3V5fX6+ioiKtWbPGJ0UBANqXT+b83333XUVERGjo0KEtttm+fbvi4uL02GOP6ZNPPvHFsACANvLJVT23bNmixMTEFrdPnjxZM2bMUEBAgD744APNmjVLO3bsUEhIiNdjhIUF+6JUAOjywsN7Wu7DcvhXVlbq448/Vk5OTottwsPDPct33XWXHA6Hjhw5ottvv93rcaqr6+RyuS3VCqD9+CKQ4J2qqtortrHZ/C570mx52qegoEDR0dGXPYuvrKz0LB86dEjHjx/XoEGDrA4NAGgjr878Fy9erOLiYn399deaNm2aevfure3bt0u6FP4LFixoss/06dOVlpam4cOHa+nSpTpw4IBsNpsCAgKUk5PT6NUAAKBj+bnd7i4xl8K0D3B1Cw/vyS95dYB105ZdHdM+AICuh/AHAAMR/gBgIMIfAAxE+AOAgQh/ADAQ4Q8ABiL8AcBAhD8AGIjwBwADEf4AYCDCHwAMRPgDgIEIfwAwEOEPAAYi/AHAQIQ/ABiI8AcAAxH+AGAgr8I/OztbTqdTQ4YM0eHDhz23O51OxcTEKD4+XvHx8Xr//feb3f+7777TM888o/HjxysmJkZ/+tOffFM9AKBNunnTaOzYsXr44Yc1ZcqUJtuWL1+uyMjIy+6/Zs0aBQcH65133lFJSYmmTJmi4uJiBQUFta1qAIAlXp35jxw5Ug6Ho82D7Ny5U8nJyZKkgQMHatiwYXrvvffa3B8AwBqvzvwvZ86cOXK73br11lv17LPPqlevXk3anDhxQv369fOsOxwOVVRUWB0aANBGlsJ/w4YNcjgcqq+vV1ZWljIzM5Wbm+ur2hoJCwtul34BoKsJD+9puQ9L4f/9VJDdbldKSopmzpzZbLu+ffvq+PHjCg0NlSSVl5dr1KhRrRqrurpOLpfbSrkA2pEvAgneqaqqvWIbm83vsifNbf6o59mzZ1Vbe6kAt9utHTt2KCoqqtm2MTExys/PlySVlJRo//79Gj16dFuHBgBY5FX4L168WGPGjFFFRYWmTZume++9V9XV1XrooYcUFxen2NhYHTt2TBkZGZ594uPjVVlZKUl6/PHHdfr0aY0fP15PPvmkMjMzFRzMNA4AdBY/t9vdJeZSmPYBrm7h4T316NqnO7uMa966acs6d9oHANB1Ef4AYCDCHwAMRPgDgIEIfwAwEOEPAAYi/AHAQIQ/ABiI8AcAAxH+AGAgwh8ADET4A4CBCH8AMBDhDwAGIvwBwECEPwAYiPAHAAMR/gBgoG7eNMrOztbu3bt1/PhxFRUVKTIyUqdOndJzzz2n0tJS2e123XDDDcrMzFRoaGiT/efOnau9e/cqJCRE0qUfdJ85c6Zv7wkAwGtenfmPHTtWGzZsUL9+/Ty3+fn5KTU1Vbt371ZRUZEGDBig3NzcFvt44oknVFhYqMLCQoIfADqZV+E/cuRIORyORrf17t1bo0aN8qyPGDFCJ06c8G11AIB24ZM5f5fLpU2bNsnpdLbYZu3atYqLi9OsWbN09OhRXwwLAGgjr+b8r+SFF15Qjx49NHXq1Ga3z549W+Hh4bLZbNq6datSU1O1Z88e+fv7ez1GWFiwL0oFgC4vPLyn5T4sh392dra+/PJLrVy5UjZb8y8kIiIiPMsJCQlasmSJKioqGr2HcCXV1XVyudxWywXQTnwRSPBOVVXtFdvYbH6XPWm2NO2zdOlSffrpp1qxYoXsdnuL7SorKz3L77//vmw2W6MnBABAx/LqzH/x4sUqLi7W119/rWnTpql379565ZVXtGrVKg0cOFCTJ0+WJPXv318rVqyQJMXHx2v16tWKiIhQenq6qqur5efnp+DgYL322mvq1s0nM04AgDbwc7vdXWIuhWkf4OoWHt5Tj659urPLuOatm7as86d9AABdE+EPAAYi/AHAQIQ/ABiI8AcAAxH+AGAgwh8ADET4A4CBCH8AMBDXWMBVJeRHdnWzB3Z2Gde0i/Xnderb+s4uA52M8MdVpZs9UP/MSe3sMq5ptz73hiTC33RM+wCAgQh/ADAQ4Q8ABiL8AcBAhD8AGIjwBwADEf4AYCDCHwAMdMXwz87OltPp1JAhQ3T48GHP7ceOHVNycrImTJig5ORklZSUNLt/Q0ODFi1apHHjxmn8+PHavHmzz4oHALTNFcN/7Nix2rBhg/r169fo9oyMDKWkpGj37t1KSUnRwoULm92/qKhIpaWlKi4uVn5+vvLy8lRWVuab6gEAbXLF8B85cqQcDkej26qrq3Xw4EHFxsZKkmJjY3Xw4EHV1NQ02X/Hjh1KSkqSzWZTaGioxo0bp127dvmofABAW7Tp2j7l5eWKiIiQv7+/JMnf3199+vRReXm5QkNDm7Tt27evZ93hcKiioqLVY4aFBbelVADNCA/v2dklwAJfHL8uc2G36uo6uVzuzi4D7YxQ6hhVVbU+75Nj13G8OX42m99lT5rb9Gkfh8OhyspKNTQ0SLr0pu7JkyebTA993/bEiROe9fLycl1//fVtGRYA4CNtCv+wsDBFRUVp27ZtkqRt27YpKiqqyZSPJMXExGjz5s1yuVyqqanRnj17NGHCBGtVAwAsuWL4L168WGPGjFFFRYWmTZume++9V5L0/PPPa/369ZowYYLWr1+vRYsWefaZPn269u/fL0mKj49X//799Ytf/EKTJk3SU089pQEDBrTT3QEAeMPP7XZ3iYl05vzNEB7ekx9zaWe3PvdGu835P7r2aZ/3i8bWTVvWeXP+AICujfAHAAMR/gBgIMIfAAxE+AOAgQh/ADAQ4Q8ABiL8AcBAhD8AGIjwBwADEf4AYCDCHwAMRPgDgIEIfwAwEOEPAAYi/AHAQIQ/ABiI8AcAA3WzsnNZWZmeeuopz3ptba3q6ur00UcfNWqXl5enjRs3qk+fPpKkW265RRkZGVaGBgBYYCn8+/fvr8LCQs96VlaWGhoamm2bkJCg9PR0K8MBAHzEZ9M+9fX1KioqUmJioq+6BAC0E5+F/7vvvquIiAgNHTq02e3bt29XXFycHnvsMX3yySe+GhYA0AaWpn1+aMuWLS2e9U+ePFkzZsxQQECAPvjgA82aNUs7duxQSEiI1/2HhQX7qlTAeOHhPTu7BFjgi+Pnk/CvrKzUxx9/rJycnGa3h4eHe5bvuusuORwOHTlyRLfffrvXY1RX18nlcluuFVc3QqljVFXV+rxPjl3H8eb42Wx+lz1p9sm0T0FBgaKjo1s8k6+srPQsHzp0SMePH9egQYN8MTQAoA18cuZfUFCgBQsWNLpt+vTpSktL0/Dhw7V06VIdOHBANptNAQEBysnJafRqAADQsXwS/rt3725y2+uvv+5Zzs7O9sUwAAAf4Ru+AGAgwh8ADET4A4CBCH8AMBDhDwAGIvwBwECEPwAYiPAHAAMR/gBgIMIfAAxE+AOAgQh/ADAQ4Q8ABiL8AcBAhD8AGIjwBwADEf4AYCDCHwAMRPgDgIEs/4av0+mU3W5XYGCgJGnOnDkaPXp0ozbfffed5s2bpwMHDsjf31/p6em6++67rQ4NAGgjn/yA+/LlyxUZGdni9jVr1ig4OFjvvPOOSkpKNGXKFBUXFysoKMgXwwMAWqlDpn127typ5ORkSdLAgQM1bNgwvffeex0xNACgGT45858zZ47cbrduvfVWPfvss+rVq1ej7SdOnFC/fv086w6HQxUVFa0aIyws2BelApAUHt6zs0uABb44fpbDf8OGDXI4HKqvr1dWVpYyMzOVm5trubD/VV1dJ5fL7fN+cXUhlDpGVVWtz/vk2HUcb46fzeZ32ZNmy9M+DodDkmS325WSkqJ9+/Y1adO3b18dP37cs15eXq7rr7/e6tAAgDayFP5nz55Vbe2lZyC3260dO3YoKiqqSbuYmBjl5+dLkkpKSrR///4mnwgCAHQcS9M+1dXV+vWvf62Ghga5XC4NHjxYGRkZkqT4+HitXr1aERERevzxxzV37lyNHz9eNptNmZmZCg5mDh8AOoul8B8wYIC2bt3a7LbCwkLPco8ePbR8+XIrQwEAfIhv+AKAgQh/ADAQ4Q8ABiL8AcBAhD8AGIjwBwADEf4AYCDCHwAMRPgDgIEIfwAwEOEPAAYi/AHAQIQ/ABiI8AcAAxH+AGAgwh8ADET4A4CBCH8AMJCln3E8deqUnnvuOZWWlsput+uGG25QZmamQkNDG7WbO3eu9u7dq5CQEEmXftB95syZVoYGAFhgKfz9/PyUmpqqUaNGSZKys7OVm5urF198sUnbJ554QlOnTrUyHADARyxN+/Tu3dsT/JI0YsQInThxwnJRAID25bM5f5fLpU2bNsnpdDa7fe3atYqLi9OsWbN09OhRXw0LAGgDS9M+P/TCCy+oR48ezU7tzJ49W+Hh4bLZbNq6datSU1O1Z88e+fv7e91/WFiwr0oFjBce3rOzS4AFvjh+Pgn/7Oxsffnll1q5cqVstqYvJiIiIjzLCQkJWrJkiSoqKtSvXz+vx6iurpPL5fZFubiKEUodo6qq1ud9cuw6jjfHz2bzu+xJs+Vpn6VLl+rTTz/VihUrZLfbm21TWVnpWX7//fdls9kaPSEAADqWpTP/I0eOaNWqVRo4cKAmT54sSerfv79WrFih+Ph4rV69WhEREUpPT1d1dbX8/PwUHBys1157Td26+WzGCQDQSpYS+Cc/+Yk+//zzZrcVFhZ6ltetW2dlGACAj/ENXwAwEOEPAAYi/AHAQIQ/ABiI8AcAAxH+AGAgwh8ADET4A4CBCH8AMBDhDwAGIvwBwEDX3NXVevbqru6BAZ1dxjXv3PkLqj19rrPLANBG11z4dw8MUMpzGzq7jGvexpwpqhXhD3RVTPsAgIEIfwAwEOEPAAYi/AHAQIQ/ABiI8AcAA1kO/2PHjik5OVkTJkxQcnKySkpKmrRpaGjQokWLNG7cOI0fP16bN2+2OiwAwALL4Z+RkaGUlBTt3r1bKSkpWrhwYZM2RUVFKi0tVXFxsfLz85WXl6eysjKrQwMA2sjSl7yqq6t18OBBrV27VpIUGxurF154QTU1NQoNDfW027Fjh5KSkmSz2RQaGqpx48Zp165dSk1N9Xosm83P67b/FxLk/Z1Am7XmmLSGvVdYu/SL/2qvY/d/waFXbgTLvDl+V2pjKfzLy8sVEREhf39/SZK/v7/69Omj8vLyRuFfXl6uvn37etYdDocqKipaNVZIKwJ9+byEVvWNtgkLC26XfofPyG6XfvFf7XXscpMy2qVfNOaL48cbvgBgIEvh73A4VFlZqYaGBkmX3tg9efKkHA5Hk3YnTpzwrJeXl+v666+3MjQAwAJL4R8WFqaoqCht27ZNkrRt2zZFRUU1mvKRpJiYGG3evFkul0s1NTXas2ePJkyYYGVoAIAFfm63222lg6NHj2ru3Lk6ffq0evXqpezsbN14442aPn260tLSNHz4cDU0NCgzM1MffPCBJGn69OlKTk72yR0AALSe5fAHAHQ9vOELAAYi/AHAQIQ/ABiI8AcAAxH+V+B0OhUTE6P77rtPEydO9OqidPHx8Tp37sq/b7tv3z7FxsYqISFBH374oS/KhRecTqd+/vOfe76fIklvv/22hgwZovXr12vTpk1at27dFftZt26dqqur27FS/NCQIUN05syZRreNGjXKq+uETZ8+XaWlpZKkkpISJSQkKCEhQX/84x/bpdau4Jr7Aff2sHz5ckVGRurw4cN64IEHNGbMGEVERLTYvrCw0Kt+CwsLlZCQ0KprHEnSxYsX1a0bh86KPn366K9//auio6MlSQUFBRo6dKgk6cEHH/Sqj9///ve68847FRbWumsRcfw63uuvv+5ZLi4u1s0336yMjNZdiuJaO27Xzj3pAJGRkerVq5cqKyt15swZvfjiizp16pQuXLigRx55RImJiZIunaHs27dPQUFBcjqdio+P1969e1VVVaXHHntMU6dO1RtvvKGdO3eqe/fuKioqUn5+vg4fPqysrCydPXtWPXr00IIFC3TTTTeprKxMiYmJeuCBB/Thhx9q0qRJcjqdWrx4secS2rGxsXryySdVV1enJUuW6PPPP9f58+c1atQozZs3z3P9JVxy//336+2331Z0dLS++uornT17VpGRkZKkvLw8nT17Vunp6Xr11Vd18OBB/e53v9N3332nSZMmac6cOTp48KBOnjyptLQ0BQYG6uWXX9Ybb7yhYcOGaerUqZKkuXPnetbnzp0rf39/HTt2TGfOnFFhYaEKCgq0ceNGNTQ0KDg4WM8//7xuvPHGzvyzdGktPda+37Zy5Up99tlnevPNN+VyubRv3z7l5eXJ7XZr4cKFqqmpUbdu3TR79myNGTNG0qXH8q9+9Sv9+c9/1ujRo1VRUSG73a6SkhJ99dVXGj9+vO6++27l5eWpoqJCjzzyiB555JHO/DN4jfBvhX/+858KCQnRT3/6U02ePFkvvfSSBg8erLq6OiUmJmrEiBEaPHhwk/3OnTun/Px8lZWVKS4uTvfff79SU1P1xRdfeMKhvr5eaWlpWrJkie644w7t3btXaWlpKi4uliR98803Gj58uNLT0yVJDz30kKKjo5WXlydJqqmpkSQtWbJEt912m7KysuRyuTRnzhxt2bJFkyZN6qC/Utdw++23a+PGjfr2229VUFCghIQEHThwoEm7GTNmKDU1VX/4wx908OBBjR49WtHR0YqOjtbmzZs9rwq9cejQIa1fv149evTQP/7xD+3cuVMbNmyQ3W7XX/7yF82fP19vvfWWr++qUZp7rAUF/feikPfdd5++/PJLz5O7JCUlJWnSpElKSkrSF198oSlTpmjnzp2eKxUEBgZqy5Ytki49oR85ckRvvvmmGhoa5HQ6VVtbq/Xr16uqqkoxMTH65S9/2WjMqxXh74W0tDS53W6VlpZq2bJlKi0t1dGjR/Xss8962ly4cEH/+c9/mg3/e+65R5LUv39/9erVSxUVFU3aHTt2TAEBAbrjjjskSXfeeacCAgJ07NgxBQUFKTAwUBMnTpQknTlzRp988onnUtqSPP+o7777rv797397tp07d+6yU1Sm8vPz08SJE7V9+3Zt375db731VrPhb7PZ9NJLLyk+Pl59+/bVxo0b2zxmTEyMevToIenScfrss8+UlJQkSXK73Tp9+nSb+zaZn99/L13szWPth+rq6nTo0CHPq/Yf//jHioqK0r/+9S85nU5Jl14l/tC4ceNkt9slSYMGDVJ0dLRsNpsiIiK8GvNqQfh74fuzu507d2revHl67bXXFBIS4vXcfmBgoGfZ39+/0RuN3rruuusa/ZO3xO1269VXX9WAAQNaPYZp7r//fiUlJem2225TSEhIi+3Kyspks9l0+vRpnTt3TsHBzV9O19/fXy6Xy7N+/vz5Rtu/D37p0nFKTEzU008/bfFemCM0NFTffPON56z64sWLqqura3QtMV881v7XD49bc2O0x5gdgU/7tMLEiRN11113adeuXerevbu2bt3q2Xb06FHV1dW1ue9BgwbpwoULnk/9/O1vf9PFixc1aNCgJm2DgoJ08803N/pEyvfTPk6nU6tXr/b8A9bU1Oirr75qc13XsgEDBmj27NmaNWtWi22+/fZbzZkzR0uXLtU999yj3/72t55tQUFBqq2t9azfcMMN2r9/vyTp5MmT+vvf/95iv06nU4WFhZ7ftWhoaNCnn35q9S5d0+68807l5+d71vPz8/Wzn/1M1113XZv7DA4OVlRUlAoKCiRdehx/9tlnGjFihOV6r3ac+bfSb37zGz3wwANatWqVVq9erTVr1sjlciksLEyvvPJKm/u12+1avnx5ozd8ly1b5nl5+b9yc3O1aNEixcbGymazKTY2Vk888YTmz5/vmabw8/NTQECA5s+fzyuBFlzpAoPz589XYmKiRo4cqZtvvlmPPvqoNm3apAcffFAPP/yw5s+fr+7du+vll19WUlKS0tLSdM8992jgwIG66aabWuz3tttu0zPPPKOZM2eqoaFBFy5cUExMjIYNG+bru3jNWLBggbKyshQXFyebzSaHw6GcnBzL/ebm5mrhwoVat26dunXrppycnCZXJr4WcWE3ADAQ0z4AYCDCHwAMRPgDgIEIfwAwEOEPAAYi/AHAQIQ/ABiI8AcAA/0/DuZFRUKCJrQAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "x = ['Reinforce', 'Mixture', 'Uniform']\n",
        "y = [1., duration_r/duration_m, duration_r/duration_u]\n",
        "\n",
        "sns.barplot(x = x, y = y)\n",
        "plt.show()"
      ],
      "id": "3b67bc2c-7192-4239-8bcd-fcf56dd8245c"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0e7d0943-56bb-4e5e-8355-ac5d6fa3f01d"
      },
      "source": [
        "## Performance:"
      ],
      "id": "0e7d0943-56bb-4e5e-8355-ac5d6fa3f01d"
    },
    {
      "cell_type": "code",
      "source": [
        "def moving_average(x, w):\n",
        "    return np.convolve(x, np.ones(w), 'valid') / w"
      ],
      "metadata": {
        "id": "QnO-w9LrlTDK"
      },
      "id": "QnO-w9LrlTDK",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(moving_average(train_r, 300), label = 'Reinforce')\n",
        "plt.plot(moving_average(train_u, 300), label = 'Uniform')\n",
        "plt.plot(moving_average(train_m, 300), label = 'Mixture $\\epsilon = 0.8$')\n",
        "\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "Zfqd-VsabWCz",
        "outputId": "b431143d-b856-4ee6-b371-d0495c26e3c4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        }
      },
      "id": "Zfqd-VsabWCz",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD9CAYAAABUS3cAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdZ2BUVd7H8e/UzKSRXggkIZQQIKEXpagUAQ2ioKKADUVdV1HcVbE8ICKuwQ6KZXcFEdvaQIMiXYjSBRIIoSa09EnPJJly7/NiYHAMkMLMJCHn84q59T8T8sudc889RyHLsowgCILQqiibugBBEATB/UT4C4IgtEIi/AVBEFohEf6CIAitkAh/QRCEVkiEvyAIQitUr/DPzMxk0qRJjB49mkmTJpGVlVVrm/fee48bb7yRcePGMWHCBLZs2VJrm+3btxMXF8fy5csvu3BBEASh8RT16ed/9913M3HiRMaPH8/KlSv59ttvWbZsmcM2W7ZsoV+/fuj1ejIyMpg6dSopKSnodDoAKioquO+++wgICGDo0KFMnTrVNe9IEARBqFOdV/4Gg4H09HQSExMBSExMJD09naKiIofthg4dil6vByA2NhZZlikpKbGvf/XVV7n//vvx9/d3Zv2CIAhCI6jr2iAnJ4fQ0FBUKhUAKpWKkJAQcnJyCAgIuOA+K1asIDIykrCwMAB+/fVXysvLGTNmDJs2bWpUocXFlUhSwx9GDgz0xmCoaNQ5XUnU1TCiroZrrrWJuhqmsXUplQr8/b0uur7O8G+oHTt28M477/Dxxx8DUFZWxhtvvMGSJUsu67iXehN1CQz0vqxzu4qoq2FEXQ3XXGsTdTWMK+qqM/zDw8PJy8vDarWiUqmwWq3k5+cTHh5ea9s9e/bw1FNPsXjxYmJiYgA4fPgwBQUF3HbbbQAUFxezceNGSkpKePTRR+tdqMFQ0agr/+BgHwoKyhu8n6uJuhpG1NVwzbU2UVfDNLYupVJxyT8adYZ/YGAgcXFxJCcnM378eJKTk4mLi6vV5JOamsrMmTNZuHAh3bt3ty/v168fW7dutb+eNWsWPXr0EDd8BUEQmlC9unq++OKLLF++nNGjR7N8+XLmzp0LwPTp00lLSwNg7ty5VFdXM3v2bMaPH8/48eM5dOiQ6yoXBEEQGq1eXT2bA9Hs4x6iroZprnVB861N1NUwrmr2EU/4CoIgtEIi/AVBEFohEf6CIAjN0Mf7P+PvG54mo+CoS44vwl8QBKGZKTAa2J2/DwAfD9c8eyDCXxAEoRkpN1Xw4rYkAJ7pP4MI3zCXnMfpT/gKgiAIjbMnP43/7P8UgDDPECJ92rnsXCL8BUEQ3GRfwQECdH60825LZtkJ2nlHcMCQgYdKy37DQX49/bt924cS7nFpLSL8BUEQLkOZqZwfjq3GU6PHV+vD79k7qbZUo9foea7/E1hkKy9vf4NyUzlmyVLn8Z4bMJMI79rD5zibCH9BEIQGkGSJtMKDfJ7xDZ38YthbkHbB7UpNZczY9Gy9jqlWqOgb2osRkcPcEvwgwl8QBKFBlh74wt4T51zwx7SJJkDnx+nybG6MuZ7ewfEsTv2YdINtiBsfjTe9QxKID4qjjYcvm06lMLHzTejUHk32PkT4C4Ig1JOhqojd+fsI1gfSL7Q3P2et487YCQyJGFRr27/3vJ8KUyVlpnKC9IFoVRr7uilxt7mz7AsS4S8IglAPsiwze+urANzXfTJRvu1JjLn+kvt4a73w1jZ+LhJXEuEvCIJwCUdLMnnrj/cdlrmyC6a7iPAXBEG4iOyKXIfg7xnUnft7TEWhUDRhVc4hwl8QBOFPTpSdItQzmM8zvrXf2L2hwyi6BcQS5dsOpeLKGBhBhL8gCMJZOZV5LNi1yGHZzR1vYGTkNVfE1f6fifAXBDczVBXjo/VCq9I6LJdkCQWKKy5kWgqLZOWzg9/YXw8M60tizPUE6PybsCrXqVf4Z2ZmMmvWLEpKSvDz8yMpKYno6GiHbd577z1++uknlEolGo2GmTNnMnToUMA2xePWrVvRarV4enry/PPPEx8f7/Q3IwhNqcBo4ItD3zKhUyIBOj9e2/UuwZ5B3NZ5PGbJTHFNCV9kfEdxTQkAw9sPJcQziN+zdzI2egRfHPoOf50fT/V9FIVCwZ8n2VMoFPycuR4/D1+uatu/qd7iFUGSJT5MXcp+QwZqhYq23uGcLD9tX39b5/Fc235wk9UnyzJyVRnW0/up2f4VPrfPAo+2Tj9PvaZxvPvuu5k4cSLjx49n5cqVfPvttyxbtsxhmy1bttCvXz/0ej0ZGRlMnTqVlJQUdDodGzduZMiQIWg0GjZu3Mj8+fNZt25dgwoV0zi6h6irYbzaqHluzQLyqwqx1OPR/fryUGmRZBmzZK61LkQfxHMDn0SjVFNgNKBSKi94ddpcP7OmqiunIpeM4qOUmcpZc2LjBbfpHxTPPfFNd0NXqizGtHcV5gNn81GhIGrmUooqGp59dU3jWOeVv8FgID09nSVLlgCQmJjIvHnzKCoqIiAgwL7duat8gNjYWGRZpqSkhLCwMK677jr7ul69epGbm4skSSiVV8aNE6Hlk2SJ46Un6OAbiUqpsi/76809SZZYuOcjjpQcv+Bx4gK6cLDoMAB6tY4qSzUAvYPjQaHAW+PFiPbDKDeXs+HkFvYUpDE04ioqzZWEeYXyU+ZaAGqspovWml9VyBObniMuoAtHSzIxS2YeTriX+KBul/05tESyqQo0ugsGtkWy8Pim52ot91fpmXHkJIc9tZSqlfQrq0Yjy2iOrqfqZDYeAyehCo52ap3W/OOYj24FSw3a3uOQjaUYk5NQegei7tAPqbwAy7HtDvto+01EpfeGCuf/sawz/HNycggNDUWlsv1CqFQqQkJCyMnJcQj/P1uxYgWRkZGEhdUeh/qzzz7j2muvFcEvNAuyLHOw6DDv7fuvfdm17QaTUXSEXGN+vY7hrfGiwlzJpC63MKzdVeQZC/gy4zseTLgHvVp3wX2CCSQmPrrW8kCdP6kFB7im3WA6tIniaMlx4gK6cKw0i3be4VRba/jv/uUcLz1h/yMD8EHqUsZEj8BorsJoMTIgrC+F+fl08epCuFdowz6UZkiWZeTqcjDXoPAJsge9ZCyh8qtnQbKg6XoN5gPr8LrjNZS+wRwqOsKnB//ncJzIKjNndBruzTyNXpLpWVFT61zW7IMYv38RAK9JSSjbXP7nJ1tqMK54yf7anLHZ/m+pNBfT3mT7a3VUbzwG34VUlo8qPPayz30xdTb77N+/n2eeeYZVq1bZl91www289tprdO/evdb2O3bs4Omnn+bjjz8mJibGYd2qVatYuHAhn332GUFBQU56C4LQeMmH1rNs7zd1b/gXL1wzgzCfECRZItQryO3NBDnl+cz/dSF39ZpIx4AoHvnx+Ytu+8/BD7EyYw0jY4ZwXczVbqyycWRZpmL/Zgp+fBfPTn0xHtnpsN7/mjvxH3Ir5qIcTr3/qH25BHwZ6kuQ2cohTy1ndLbhFHRWicGlVQyJH4P+12/t2weMuBtLaSFtBo1DoVQDCgp/eh8USodz+l19C35XT6Row6f49huL2icAhYdng37m+SvfoWK/LfD1MT2pOm7rQurZZQD6DgmU7foZfXQ8nh374Nm5b0M/skapM/wNBgOjR49m+/btqFQqrFYrAwcOZM2aNbWu/Pfs2cMTTzzB4sWLa/1hWLt2LUlJSSxdupR27Rr+dJxo83eP1lLX4eJj/JK1gYziIwA83vtBOrSJZt2JTWzN2UlCUHcmdh6HjIyhqphgz0AASmvKKagqpJNfB5fU1ViFVQbmb3+TAWF98NR4kl2Ry/GyLIzmKoftwjxDaOsdxn3dJzdZf/VLfWbVv3+Gef/aOo+h8ApAriyiTKXkdMcEVD4hLCndW2u7wSVGxkt+eN36MgCW7INYTuxFFRSFutNVDgH+57qkCgNVP72OVJJz0RqU/u3QXfcAqqBopOpyarb9D1VQJJruI1D86bOVKoqo/PxJFD7BeN2xAIVCgVRhsB3DO7DO99rY/2N1tfnX64bvXXfdxa233mq/4fvNN9/w6aefOmyTmprKjBkzeOedd+jZs6fDuo0bNzJv3jyWLFlCVFRUg98EiPB3lyupLkmW+CM/lW05u/DV+nBzpxvw0Xjzy4mN/Hh8tX27Fwc9Yw93d9TlLsHBPiRt/IDd+fu4M3YC3xz50X4D2c+jDTfFjKFvaE/Uytqtv+diwRXfaP76mcnVFVR8+TSqkBisp/fbl2t73oBsrkGh8UCTMAYsNSg8vKn85gXkCgNW4LXYSEqs1Q7HD9EFYJYlnuzzMN5Hd6GO6oXSN6TBdQFIZflUfvkMKBQgSxfcTxXZC+vJ8394NAlj0Q2aBIC1+AzGr23fyrzufA2lT3CdddSnrvpwSvgfO3aMWbNmUVZWhq+vL0lJScTExDB9+nRmzJhBfHw8EydO5MyZM4SGnm8fW7BgAbGxsQwaNAiNRuPwTWHp0qX4+9e//6wIf/doyXVZJSsmyYShqpjXdi3CIlsvuq1erWdM9HCifSPtV/GuqutyyJIFU+pqNF2GoPT0u/S25hpqdn2Hun08qrZxhIT6kZtXwsnyM0T7tgcgNXs3Hx1ybAd/KP4eEoLPf1O3Fp7A+N2c2idQe6D0j8Bz3CwUam3t9fV07jOTayqp2fkt5vQN9nUK70D0Ix9BGRiFQnXhW5JyTSXf7fiIDebzV+UR3uFM73F3o/+I/7mui5Elq+0PkNYTqcKAKXV1vb6lAKg79EM/6tG6N2xEXRfjlPBvDkT4u0dLrSu7IpdXdryFTO3/I15qT7y13uT96QbuoutedUqzhys/L9liwvj9S0jFtj7omthhWE7uRa4qQxN3LR6D7sD4/UuoOw1E4elHzeYlDvsr9d5IVRW1jmsBfvPzxKRUsD7ANuLkjJNFtDVZUPgEU2o0UK1UkKXXUKxWMbqo8nxNgDqiG/rRj6O4wFj0sizbnlGQrKBQ2r85yOZqzIdTUHccSFCQDzkpqzDt+Nq+n8LTD23CaNSdB6PU+170M8koOsKivf+2vx4ROYybO97QZD9L2WS0/RFI34jn+OdBqabyuzlQc/4z8xg0CW3CWLfWBSL8W2yYNZXmXlduZR7pRYe5JuJqzJKZTad/48fjvzhs66X25PE+DznMiCTJEr9l7yC14AAPJ9xr787prLqc7aJX3/WgatfDofnkUrb56lgRYgvb5zMLydRr+DysTa3trgntS6lUw+m8dB7OysNbkvG6YwFK3xBOlp3mg93v0724lN/9PGlbbWZkUSXZHmoGBMdTETcE36N/oD+wkb9+6pruI/AYcDsKTd2TmuzK3cOS9C/sr18dMhsf7cXDraGc9bOUayqxnNiLQueNXFNZ696Cu+oS4d/Mw6y5ac51fbzta1afsDURJAR1J7XwgMM2TdHX3RmflzX/uK0boEKFfuxMVKGdqVjyEACa+NHorroT2VyDceXLqKP7oO11A9Wbl2A5us22TbcRSKW5aLoNRx3VG4VSiVRhwLL5IyS/KNTRvVEFRqLwcBxXXqoqQ6n3ZcfxjXyS9XOtuuKDupFWmH7RugeWGgmKHsiq4gtPY/hXIw0VjCw22l6oPdB0HIDHsGm1glGSJSySlTUnNtDGow1qhYp1pzaTW5kHQBe/jjzWe7rTb1g35//7IvxF+Ltcc6zLKln5LusHNmVtBUCpUCKdvfl2dXh/ro8aflltvZfjcj4v87HtVK9//6LrNT1Gobt6ykXXyzWVoL14l8OG1Ja0c6F9iINHez1AXEAX+zpJljhYdJicyjzUCjVfH1lZa/9BFVZOBIcyPnIEHxz6CrBNXVhudmx2mhV7Jwmde3GypJLimpJa4+JXmo3M2/465abazVUAEzolcl37IS7pqdQc/++DCH8R/m7SHOt6+48P7E/U/r3n/bT1DuOD1KUMi7iaq5t4nJuGfF6ypQbT3lWowmIxZ/yK5fgO+zrP8S9gOZOOadd3AGi6j0Q3eKrbaquyVPHdkVUMCOtDZ/+YOrcvMxbx2fb32C+XMxx/Jlz71Nm+8hf25wlROgVEU1pVTkGVwb5+WvcpbM3Z6fDg2rknpDv4RtI3tBfXtLvapd1Tm+P/fRDhL8LfTZpTXZIs8eWh7/gtewcDInpxT+zkpi6pllrdFs/9Osky5oxNyJXFWHNtgWbNOVRrf23vcWj7jrcHpyxZUTTz+xGNtSpzrX34iksZFN6Pu+Jud0NFjprb53WOq8JfDOksNEt/vlJUKpTMGHQfpcW1H8VvauVpmzBXmNF0GoQsy1SvX4xsLEUZFFVnN0Cvqe+g9HS8ueqs4G+ObuwwihHth/HB/v8S7R3NTR3HcLo8m1xjPptP/46HyoNHek5z2s144dJE+AvNQnZFLifLT5NuOES4VyjJmWsA6B/am1s734RWrQWaR/hL5YUYf3odpXcg5WdsN52rN3wAChWcfbbg3NW+OmYAUoUBhUqDwsML3bD7UOic10OlpdGpPZg/6hn7lWykbzsifdsxIKxPE1fW+ojwF5pctaWa+TverLX8nm53NKtQsOQcwrx/LZbMXQBYS3MdN5CtoPbAo98ETOnr8Rz7D6cMCiYIriDCX2hS+cZC5m5bYH8d6dOOKN/2TOyUiEalcXs9prQ1KLwD0XSwDa4lSxaq1izCenKffRuFVwAK7wBkYynBo+6h0rMd1vxjSMXZqNvGoQrthDZhtNtrF4SGEOEvNIlyUwVv/rGYfGMhYOu7/Xifh5qsHsupVKp+Pv/tQzlhLsrA9lQlL7A34QCoOw9Gd800e9u8d7APVQXlKKP7QrR7RmMUBGcQ4S+43V8f0Q/Q+fNQwj1uObdUWUzVmoWowrrYHp6yWqjZ8TXmNMenhP/8dK06qjceV09B6SOGIReuHCL8Bbc5VpLFm38str/uG9KTxJjrCfFs+EiHjSFLVqqSk5BKc5EKMpGNJQ4zJ+lG/h11dG+qfnoDa/bB88uvexCFVu+WGgXBXUT4C25xrpkHQKfyYFb/J9z+VK71dBpSaS7aAbdh2vG1PfgVnn543jIHpZdtlFnPxGeQZYmarV+gCukogl+4IonwF5xOlmU+z/iGrgFdOF6axabTv9nXDWk7kImdb0Lrhpu5siQhV5Uileai0LehavXbAGjjR6PQemI+uAFNlyFo42vfnFUolJccWkEQWjoR/oLT7crby+85O/k9x3H6veHth3JLpxvdNoNUzW/LMB/c5LBM3XkwCpUabbfr0Ha7zi11CEJzJMJfcBpZlvkgdSn7Dbb2cq1KS6g+iPt6TCFYH+iW0Lec3o8p1TZL17lhjVWRPbGe3IcmfjQeZ2dYEoTWToS/4DQ/HF9tD/4H4++mZ3APt51blmVqtn6Bef8ah+Xe9y5GofV0Wx2C0FLUK/wzMzOZNWsWJSUl+Pn5kZSURHR0tMM27733Hj/99BNKpRKNRsPMmTMZOnQoAFVVVTz77LMcOHAAlUrFM888w3XXia/cLZ1VsvLj8V/YlbcXi2yxD8P7TP8ZtYbqdSXZYqLi4wcBUAa0R9NjJKZd36PtM14EvyBcRL3Cf86cOUyePNk+gfvs2bNZtmyZwzYJCQlMmzYNvV5PRkYGU6dOJSUlBZ1Ox3//+1+8vb1Zu3YtWVlZTJkyhTVr1uDl5XWRMwrNmSzL5BnzeW3Xu1RbHcfbcWfw1+xJxrTnB7CY7Ms8J76EQqFA2/Uat9QgCC1VnY2wBoOB9PR0EhMTAUhMTCQ9PZ2ioiKH7YYOHYpeb+sSFxsbiyzLlJSUAPDzzz8zaZKtrTU6OpoePXqwefNmp74RwX3+nbaMedvfoNpaQ5+QBB7t9QAj2g/j6X6PuS34rSXZmHZ+4xD8XpOSLmu6PEFoTeq88s/JySE0NBSVyvY4u0qlIiQkhJycHAICAi64z4oVK4iMjCQsLAyA7OxsIiIi7OvDw8PJzc294L5C8yTLMpUWIy9tfY1Ki20qvucHPElbb9vP+M8zP7m0DslC5VfPIpcXgFaPbtCdKIOiUAVFueX8gnClcPoN3x07dvDOO+/w8ccfO/W4l5qUoC7BwT5OrMR5WkpdRw1ZPLcuyf66jc6Xd2+ch4da6/a6ilO+sQU/EHH3y3iERru1hgtprj9HaL61iboaxhV11Rn+4eHh5OXlYbVaUalUWK1W8vPzCQ8Pr7Xtnj17eOqpp1i8eDExMeengmvbti1nzpyxf1PIyclh4MCBDSpUzOTlHheq6+u0n+z/7h2SwLTukykrrsGd4+sH+mnJWnB+Ji/v+/9DmVINTfwZNtefIzTf2kRdDeOqmbzqbPMPDAwkLi6O5ORkAJKTk4mLi6vV5JOamsrMmTNZuHAh3bt3d1g3ZswYvvrKNqlzVlYWaWlp9p5AQvOWZyxgX+EBrms3hPeGL+CBHlPd9pDWn5XuWGX/t37csyhUopeyIFyOev0Gvfjii8yaNYvFixfj6+tLUpKtCWD69OnMmDGD+Ph45s6dS3V1NbNnz7bvt2DBAmJjY7n//vuZNWsWo0aNQqlU8tJLL+Ht3XpnM2opdubuYWn6FyhQcG37wW49t2w122bGQqb61/9iOfI7AN73vi/G2hEEJxATuDeR5lzX0dPZLNr7ETmVeYBtWIaJnce5rQbLqTSqfn6j1nLP2+aj8o+4wB5Np7n+HKH51ibqahgxgbvgFpUmI+/u/bc9+B9OuJf4oG5uO79stVC15h3HhR5ehI1/nEq/5hX8gtCSifAX7E6Vn+HVDbbg7RvSkyERg+ji39Et55ZNVVStfRfr2QnRPQbdgTZhDLIso1Ao8Az2obIZXpUJQkslwl8AIN1wiPf2/ReA69oP4ZaON6I6O1Whq8mSRPWm/9iDX9t7HJqzwyyLh7aE1kSSZbbsy6ZTOz8iglw7AoIIf+Hs+PvfAvDIgLvp7u2+AdkATHt+wJK1G1VYF1tPHhH4QitSWW1m//EiencOYtPebL5cf8S+rnO7NrzyyBCXnFeEfyu3I/cPPkn/EoDxMWO5tsNVbrvpZck9QtUP8wFQtAlDnzhLBL/gdLIsU2O2otPa4q7GbEWlVKBWub/LMkB+SRXBbXQYSqv5Yv0R9hwpvOi2R06XcuhEMW39dU6vQ4R/KyXLMpVmoz342/tEMDzSfc9eSCW59uBHqcLzxqdQKJvml1G4MpVU1LDhj9Mk/37CviyojY7C0moAwgM9+ecdvfl9fw55RVVMn5DgsP83m46x9UAuMW19MZRW88gtPQhq49jNeP9xA2qVki6RfigvcOFirDaj91CTesyAQmHrgfPmV/suWfdzd/WlU0QbJFkmr8hIQpdgl1yQifBvhb489D37Cw/ir/MD4L7uk+kV3AO10vX/HeSaStuN3bMTpGv7TUDb6wYUbji30HqYzFaefPe3WsvPBT9AjsHIP947v01KWg5tg7woLK3CZJbsy3cfsg0n8vT7W3n6zt50jfKnqsbCk+/+Ro3Zat+ua6QfD9/cAx+9BoVCwavLd3P4dOkl65w1pQ9d2tt+Dw9mFREe5IWftwcASoWC8EDXtfuL37hWZn/hQbac2QpAcU0JI9oPo19oL7ecW5YlKj75u/21tteNePS5yS3nFlqHFVuOk55VjKfOFm3hgZ5MuyEOH08NKqUSpVKBt15DXrGRlNQc1uw8hQJoG+zFmYJKqk0We/B76dSM6t+eo6dLKSipIq+4igVf7LnouTNOlvDEwpQ6axzZrx0D4kLpFNHGYXlc9IUHynQVEf6tSHZFLu+nLgHguQEz8dX64KN1z5PWUnU5lcses71QqvGc+BJKvzC3nFtoWSqqzGTmlFFWaWJwvG0MscOnSli76xT3jOmKt15Tax9jtZmft59k1dbzTTztQ7yZO23ABc/RLtibO0Z05pZhMXhoVEiyjJ+/F2UlRiRZxmyW8NA69nbbkprNkp8y7K+v79+eScM72e9TpR4r5O2vUx32efuxIVSbLIT4N79JhUT4txKyLPP14ZUATOs+mQjv2gPzuezc5mqMK162v/a+e5EYokFwIEkyy37JYPO+HIflv+/PJcdQSUmFbd6G3YcKmHp9F67rHYEsw5v/20t6VrHDPgkdA/HWa5h4Td3PqHhobAGvVCgc/62t3c15aEJb4iL9+XzdEe4aHYu/j8dfzhvER09dS2WVGZVKiVqlQKdV4+vl3tFv60uE/xUspzKPxfs+pqj6/C/HhE6J9HVTM885xlULkMvy8Bh0J9qE0W49t+BeGSeKCfHXE+Bb/94pkiSzdHUGKanngz+hYyCpxwwcPFFca/vlaw6zfM3hWsu9dGoem5hgb0N3hSA/PTNuTbjoerVKSRtvj4uub05E+F9hqi3VZFfmUm6q4KO0ZbXWD40Y5LZaLCf2ULXhQzBXo2rXQwT/FSq3yMjaXafYnZFPmdHssO6fd/Si25/ass8NJXbsTCnzP91N785BWCWZ1GMGOkW0YdbUPvZeMzsO5pF23EDi1dGcyC2nb2wwGSdLWL7mMHlFRvsxZ03pQ/sQb/QeIs4aQnxaV5DcynzmbX/dYdnV4QMYENab9KLD3BQzxm396I2rFmA9kw5ge3hr5N/r2ENoiU7lVzDn4x0XXf/6l3uZeE0Mp/Ir2HEwv9b6c33cFQp4Zkpvh+6SA+JCGRAXCkDo2Tbz7tEB/OvBQRSVVaNQKGo1vQj1J8L/CpFRdIRFe//tsOzmjjcwKupaADq7a4yemkqHHj36xGdQt41zy7kF99qWnstHP9j+wHeP9ueOEZ2JCLZ1ICgsrWJnRj5fbzzGt78ev+D+QxPC2XK2qefNvw9G1YDnPBrSrCRcmAj/Fq64uoTlB78mo9j2SPgjPafRPbArZqsZjap2rwhXkCULmGswZ+6iZvMS+3KvqW+j9HRd+6vQNCxWid/357L0Z1vPl9n39iM6zNdhm6A2esYOjCLQV8cHKw/QMcKXO0Z0pl2QN4ZKM0ZjDZ0i2jCibztUSkWLaSe/kojwb8HWntjEimPnp1h8MP5uugd2BXBf8MsyVWvexXpyr8Ny/U3PieC/Ah06WUzS5+f7uv/rwUGEBly8G+Ofm27O6RnhZ39iNTK0ec6Z2xqI8G+hckL8V5oAACAASURBVCvz7ME/s8/fiPSJQKtyX5ey6s0fY87Y7LjQwwt1VB/0197vtjoE1zt6upQ3/7eXpyf35vN15wcde/mBgZcMfqF5q1f4Z2ZmMmvWLEpKSvDz8yMpKYno6GiHbVJSUnjzzTc5fPgwd911F88884x9ncFg4NlnnyUnJweLxcLAgQN54YUXUKvF357G+uH4LwA8028Gkb7t3HZeqaqMyk9nOCxTePrhNflNMTbPFchYbebtr/dRbbLy0tJdAIzo044p13dp4sqEy1Wv39Y5c+YwefJkfvnlFyZPnuwwT+857du3Z/78+dx/f+2rvg8++ICOHTvy448/8sMPP3DgwAHWrFlz+dW3UoVVBvYV7KdvSE/3Bn9lMVU/vGJ/7TlhLl53vIb31LdF8F9BzBbb8AZrdp7i0be3YKyxcNPgaPv68UM7NFFlgjPVeeltMBhIT09nyRLbjbzExETmzZtHUVERAQHn++9GRUUBsG7dOkwmk8MxFAoFlZWVSJKEyWTCbDYTGurYDihc2unybP61822HZSOjrnH5eaXyQqp//S9yjRHJcALUWnTX3I+601UoVOKbW0snSTJfrD/C/swiZt7ek1N55SxesZ8/z+w9sFsoNw+N4eahMU1XqOB0df725uTkEBoaikple9xZpVIREhJCTk6OQ/hfyiOPPMJjjz3GkCFDqKqqYsqUKfTt27dBhV5qIuK6BAc3z5tK9a1LkiT+sfl9h2VTe95C3xjXdKE8V1fpjmQMa5c4rAu95Um8uvR3yXnr0tJ/jk2hrtq+WnuI9btPAzDrg6211r8+YyhdIv2d/nxIc/3MWlNdbrl0W716NbGxsXzyySdUVlYyffp0Vq9ezZgxY+p9DIOhAkmS697wL4KDGzfzvas1pK4DhkNUW2oYGXkNCUHdUSqUdGgT6ZL3da4uy8lUqs4GvyZ+NNqeY1HofDAqVRib4PO8En6O7nap2sqNJv634Si/7c8FIPHqKPu49/eO7cqQ+HCUSlvgFxZWuK2upnSl1aVUKi550Vxn+IeHh5OXl4fVakWlUmG1WsnPzyc8vP4Dgy1fvpxXXnkFpVKJj48Pw4cPZ/v27Q0K/9bq58z1JGf+gr+HH4kdrndLF05ZslKz9XMAvO9+F4XOPSN/Cq4nyTJHTpXw1v/2YTrbtn9u8pCbBndAkmQ0aqWYUa0VqDP8AwMDiYuLIzk5mfHjx5OcnExcXFy9m3wA2rVrx+bNm0lISMBkMrF161ZGjRp1WYW3BkZzFcmZtl49D8bf7ba++6bdK5BKc9GNelQEfwslyzK/7cvmg+/20S7Em0HdQvlP8kGHbUL89My8vae9u6ZapYTag1kKV6h6Nfu8+OKLzJo1i8WLF+Pr60tSUhIA06dPZ8aMGcTHx7Nr1y6efPJJKioqkGWZVatWMX/+fIYOHcpzzz3HnDlzGDduHFarlYEDB3L77be79I21ZJIssfFUCj+e7c45rftkt/XqMRWcwrTnRxTegaij+7jlnIJzWawSD762yf66pKKI/ceLHLbpEO7DzNt7XXBsfKF1UMiy3PCG9CbQWtr8T5SdYsGuRfbXwyKuZlLszS6vR5Ykqn5+A+uZAwDob3wadUQ3l5+3vlraz9Hdisqq2bwvmz8OF3K64HwbfddIPzq38yP1uIGbh3QgItir1jy07tZcPrO/utLquuw2f8F99hcetM+05aX2ZErcbcQHuXZQNMlYSs3vy7Ec32lf5nHV5GYV/MKlbT2Qy79/THdYFhnqzbtPDbffrL1lmOimKTgS4d8MmKwm3v7jQ06UnwJswzDfHnszGhdOai5XV2BclYRkOGVfpom7lrZj76OookV8GWy1Mk4Us+CLPdw7tit7jxSy9+j5YZEnDIshItibblHO754pXFlE+DcDO3P32IN/evzd9Aru4dLzyZKFimWP2l+rYwbgMeQulDofVHpvqGh+X30FmxqTlfe+TwOwj6oJ8OTtPekRE9hUZQktkAj/JibLMluytwEwf/Dz+Hm0cfk5zQc22P/tfc97KDy8XH5OofGyCyt54T/bHZZd1T2MrQdsffQXPTEUL524cSs0jAj/JmSVrLy++11OlZ9hcuxEtwS/LEmYD29B4RuK9x1JLj+f0HAVVWZO5VcQG+lHUVk1C79NdVjfNdKP6eO6MX2cuC8jNJ4IfzeTZAmLZEWWZd76431Olp+hrVcY/cNc261SliyY96/HfGgzUvEZdMMfdun5hIbbdiCXj/5y4/ach27qjslspUukH8FN3FtHuDKI8Hezb478wK+nf7e/1ig1PDvgCZQK146KaTm+k5ptXwCgiuyFuuNAl55PqD9Zlvl83RH7GDt/NWZAJAO7iYEQBecS4e9G1ZbqvwS/mleHzHZ58EtlBVRv+BCFzgePq+5EHd1X9ARpRpb9cohf92YD0K9rCH8b3x2FQoHFKtmeuhUEFxDh70YbTm0B4PHeD2JUVhDr1RWd2rVzl8qyjHHlywB4DL0HTYd+Lj2f0DC5RUZ+3ZuNQgEv3N2P6DAf+x9mEfyCK4nwd5NdeXtZlbmWboGxdPbrSEiIr1ueJrSeOYBcVYq64yAR/M3QD79lolYpSHr4avx9xCTmgvuI8HeDzNITLDlgGyVzStdb3dbkYj66jeoNH4BGj+6a+9xyTqH+sgsr2Z6ex+gBkSL4BbcT4e9ilWYjr+9+D4AZvR50S3dOAKvhlC34Af2Ih1C4uHlJqD+zRSIzp4xPVmegVasYMzCyqUsSWiER/i60Jz+N5LMjc/4t4T5iAzq55bzm4zuoXrcYAM9b56EKaO+W8wp1qzFZWfhtKgdPFANw54jO+Hpqm7gqoTUS4e8CJ8tOk7Rrof31gLA+9HDxAG3nWPOPnQ/+m54Xwd/MPL5oCyazbRKVsYMiua5PRBNXJLRWIvydbGv2TpZnfG1/HaQP5PYu491ybslYgvGnNwDQ3/BPVGGd3XJeoX427TmDySyhAN7/xzVoNWLmFKHpiPB3okqz0R78N3e8gRGRw1zeh/8cWZap/vVjsJrRj34CdTvXDg4n1I/ZYuVEbgWeOjWfrztMUBsdL90/QAS/0ORE+DuBRbLw2q53OV1he1Dnwfi76enikTn/ynxwI9ZTqXhcPRV1VC+3nlu4uC/XH2XjnjP213+/JR6dVvzaCU2vXpelmZmZTJo0idGjRzNp0iSysrJqbZOSksKECRPo0aOHfZrHP/vpp58YN24ciYmJjBs3jsLCwssuvjkwWc18lLbMHvxttL5uD34Ac/oGlIHt0XQf7vZzC+dJksy5yfEMpdX8lpZjX6dWKWgfIuZEFpqHel2CzJkzh8mTJzN+/HhWrlzJ7NmzWbZsmcM27du3Z/78+axevRqTyeSwLi0tjXfffZdPPvmE4OBgysvL0Wpbdg8HWZZZd/JXVhz7CYAhEYPw0XgzwMUDtF2ItSALqeg0HldNRuGmZiahthqTlTkf7yC/pMq+TK1SsOBvVzX51ImC8Fd1hr/BYCA9PZ0lS2zTCyYmJjJv3jyKiooICAiwbxcVFQXAunXraoX/0qVLmTZtGsHBwQD4+Pg47Q00BaO5iqe2zLG/vrbdYG5z003dCzFn/AoKBeqOA5qshtYsv9jIJ6sP2btv/tktQ2NE8AvNUp3hn5OTQ2hoKCqV7QaVSqUiJCSEnJwch/C/lGPHjtGuXTumTJmC0Whk1KhR/O1vf2vQk66Xmoi4LsHBzvtjI8sy//zlbQB6hXXjhi4j6BXeuHHVL7cu2WLGXJRN+aEteHbsQ2iUc7p1OvPzcqbmWtfn64/agz8i2IvHbu+NUqEgrkP9fj9cqbl+ZqKuhnFFXW6582S1Wjl06BBLlizBZDLxwAMP0LZtW26++eZ6H8NgqECSGj63bHBw42a+v5jVWes5VZpNlE97pne7F6BRx7+cumRzNcbkJKSCzPPLYkc45X06+/NyluZaV2GFmdSjhQxNCMfXS8vwPu3w97E1aTZ1vc31MxN1NUxj61IqFZe8aK6zgTg8PJy8vDysVitgC/L8/HzCw8PrXUTbtm0ZM2YMWq0Wb29vRowYQWpqat07NjMnyk7x4/Ff0Cg1PNHnoSarw/jDKw7Brxv+EOoIMauTu5VWmnj6XdtIrROu6cjEazqKMXqEFqPO8A8MDCQuLo7k5GQAkpOTiYuLq3eTD9juE6SkpCDLMmazmW3bttG1a9fGV90ErJKVN/94H4BZ/R9Hq3L/DWtZlqnZvQLJcBIA7/v/g8+DS9F0usrttQiw/JdDAAzvE0Ebr5bdgUFoferVNeTFF19k+fLljB49muXLlzN37lwApk+fTlpaGgC7du1i2LBhLFmyhC+//JJhw4axZYvtqujGG28kMDCQG264gZtvvplOnTpx6623uugtucaqzLVYJAujo4YT5hXi9vNLxhKM/3sW0+4VAHhOeBGFSvQXbwo1Jitrdp5i9+ECAKaM6tLEFQlCwynkc52Sm7mmbPPfcmYbXx76jnCvUJ4f8KRThmRuSF2yZKXiP/fbX3ve9LzLhm640to9nUmWZd76eh/7jxfZl7331HXoVc1zVrTm8JldiKirYVzV5i8uHS+hpKaUj9KWcaLsFAD3dZ/cJNMfWnMPA6Du0A/diEdQKEVf/qaw90ihPfiH9WzL3aNjCQ11z6Q8guBsIvwv4lR5Nq/ufNv++vkBT9LWO8ztdcgWE6Zd34NGh+7a6SL4m8j63af5bO1hwgM9een+AajEz0Fo4UT4X0Cl2ch7e/8DQCe/Dtze5eYmCX6Amq1fYM09jG74wyg0oieJu5gtEhq1ksLSKp5+f6t9+fX924vgF64IIvwvYHXWesrNFUztehtXte3fJDVIxhKqN/4b65kDqDsPRtNpUJPUcaU5dqaU8EAvUo8XcuxMGTcMinLonpmZU8aX649w5HRprX0nj+zM0IS27ixXEFxGhP9fbM3ZxYZTW+gf2qfJgt9akIXx+xftr7Vx1zZJHVeS/ccNfLH+CDkGo8Py9btP8+Sknhw8UczP2046rAsL8CS3yMhd13dhcHy4GIZZuKKI8P+TvflpfHbwa/w82nBrl3FNUoNcXWEPfqV/O7Q9x4pJWS7TtvRcPvoh3f5apVQQ4q+nU0QbtqTm8OZX+xy2Dw3wZO59/UXYC1c0Ef5n7crby5IDn9PBN4pHez2ArgkmPJdNVVR89QwA6i5D0F/7gNtruJIUl9fw9Pu/Yz3bRfjBcd0Y2C3UocfW6AGRvPCf7QS10THnvv546TRNVa4guJUIf2zDNiw58DkAd8Xd1iTBb80/jnHFSwCoO18tgr8RLFaJjXvOsO1AHpk5ZQ7rnr6zN12j/Gvt0zbIi49niTkQhNanVYe/WbLwafpX7M63fe2/t9udhDbB07uWrD1UrXkHAIWnH7qrp7i9hpbsZF45r3y6G5NFqrWuX2wwDyR2E004gvAXrTr8t+fssgf//w38B2FeoW6vwZJ9kKq176LwCUY/+nFUAe3cXkNL8+veMxw+VUq/rsEs+jbNYd11vSO4qkcYSoWC8EBP9B6t+r+4IFxUq/7N+PH4L/h7+DHv6mfd/uSuLMtULH8C2VgCgO6qySL462HZ6gw27bVNmbn1QK59+X1juxIT0YaIIK+mKk0QWpRWG/578tOoMFcyvP3QJhmywXhox/ngH/EI6ujebq+hJdl3pIAXPvjd/rp7tD8V1RZuGBRFv9jgJvkZCkJL1irD/8tD37PlzFY0SjU3drje7eeXygrI/3EhqD3wvnshiia4wdxSyLLMsl8O8evZq324+M1bQRDqr9WF/2cHv+H3nB2AbVz+pujZU7PrWwA8xz8vgv8iJElm7a5TfLPpmL2rZrdofx6bmICHuHkrCJetVYW/yWqyB/+cQU8R4hns9hqk8gIsx3fi228scmCk289/MdUmC2t3nmLiyNimLgVZllm1NYvvt9hmK+vbJZiHb+2JSqrdm0cQhMZpNeEvyRIfpn4CwGO9pjdJ8ANUp3wKkkSb/jdQYm6SEmo5kVvO3KU7Afh+Sya9Owfx9wnxKM+2o3+x7gihAXq6Rvrj7+Phsh40a3ee4ov1RxyWzZ02gPYh3gQHeomhkwXBier1W5yZmcmsWbMoKSnBz8+PpKQkoqOjHbZJSUnhzTff5PDhw9x1110888wztY5z/PhxbrnlFiZPnnzB9a70w7HVZBQfIVAXQBf/jm499znWkmysp9JQR/dB4xcKzWBykrTjBt7+2jafsqeHGmONhT1HCnn/+/32mar+6vVHriYrt5zwQE/CA53Tu6bGbK0V/K8+fBUhfnqnHF8QBEf1Cv85c+YwefJkxo8fz8qVK5k9ezbLli1z2KZ9+/bMnz+f1atXYzKZah3DarUyZ84cRo4c6ZzKG8AsWdhyZhtga+5RKppmSF7THz+AWoPH0Hua5PxgC/xNe7M5nl3K7/tzkWUI8PXgqTt6ExrgiZePjknP/+QQ/IG+HhSXm5DOTvr2z8Xne93cdm1H4jsG8q/lf1BVY+HBm7qhVirReajQqJRkF1ZitspEh/nQMcIXlVLJucnjcouMbD2QS2SID6t32AZVmzS8E4Pjw/HWi2EWBMGV6gx/g8FAeno6S5YsAWyTsc+bN4+ioiKHSdyjoqIAWLdu3QXD/6OPPuLaa6/FaDRiNBprrXeVoqoSkna+Q7W1mgd63IVK2TQ3C60l2ViObUebMBal3rdJapBlmXe/S2PPkUL7ssgQbx6dGE9QG9sVtqdOw1uPDua1L/fSpV0bukUHEBftbx/z5tykJmqVEotV4utNx/h60zH78f48gNqFxEX5c/BE8QXXDYkPZ/SA5nMfRBCuZHWGf05ODqGhoahUttBUqVSEhISQk5PjEP6XkpGRQUpKCsuWLWPx4sWXV3EDZJae5N3N/6baUsOwiKvoGdzdbef+M2vRaYzfvACAJmFMk9RgMlt5fGEKNWYrYOs58/D4Hhe8wm7j7cHLDwy84HFG9G3HdX0iUCoU5BYZee4j2zeqMQMjUasUGEqrqay24OulZc/hAlRKBR5aFQUl1QC1gr9dsBenCyoBuGds099sFoTWwuU3fM1mM//3f//Hv/71L/sfkMa41ETEF2PVBdM/oiejO11Dl6CYRp/7cpSnbqTgx3cB8Bt8KwGREfZ1wcE+bqvj6UVbqDFbUSkVfPtqIirVxZu+6ltXcLAPP74xvt41pGca2JqWw9SxcWjVSvuDWZIkI8ky6kvU1JC63K251gXNtzZRV8O4oq46wz88PJy8vDysVisqlQqr1Up+fj7h4eH1OkFBQQEnT57kwQcfBKCsrMw2tEFFBfPmzat3oQZDBdLZ/t71pULPY4Puo6CgvEl6isiyTOWmrwDQj3kSa2SCvY7gYB+31HQwq4h3vknFZJEY1D2UB8d1p6io8qLbu7KuYG8tN10VRVlJw5v93PV5NVRzrQuab22iroZpbF1KpeKSF811hn9gYCBxcXEkJyczfvx4kpOTiYuLq3eTT9u2bdm+fbv99aJFizAajW7v7dMULEd+Ry4vwGPIPagjE9x+/s37sln6cwYA1/Rqy50jxKQwgiDY1Kvby4svvsjy5csZPXo0y5cvZ+7cuQBMnz6dtDTbqIq7du1i2LBhLFmyhC+//JJhw4axZcsW11XezMmSlZrtX6H0C0cTO8St5zaZraxMybQH/5OTenLPmK5iWGNBEOwU8rl+d81cY5p9oOm+ylVv+xJz6mp0I/+OJqb2XMCuqivtuIG3/mcbptpDq2LutAEN6it/pX31dbXmWhc039pEXQ3TZM0+QsNZDScxp65GGdwBdYd+bjtvaaWJj344AEDvzkH87eYedd5EFQShdRLh7wI1O20Dt+mufcBtQw3vPVrIwm9SUSkVzHtgoBjXXhCESxLh72TVKcuwntyHtucNqPwj6t7BCf634aj9Cdmp13cRwS8IQp1E+DuRVF6AOX0DANr+E91yzv+uSue3NNuMVvOnD3TaWDuCIFzZRPg7ifnoVqo3fAiA58R5KFw8jERFlZn3V+y3PzH78gMi+AVBqD8R/k5gLTplD37tgNtRBbZ36fk27jnDd78eo7Lagt5DxVuPDhHdOAVBaBAR/k5gPvwbAF6TklC2CXXZeX7fn8P3m49jKKsBoGOEL38b30MEvyAIDSbC/zLJkoTl6DbUUb1dGvyLvk21j8bppVPz8M09iIv0R6kUE5cLgtBwIvwvk+XoVmRjCeoug112jqKyanvw/+uhQYT6e7rsXIIgtA4i/C+DNf8Y1Zv+jcInGHVUL5ecIyU1h6U/Z6BQQNJDVxEkZrYSBMEJRPg3kuVUGlWr30LhE4R+zBMolM79KPccLuCrDUfJL6kCYOI1MSL4BUFwGhH+DSSbjFT9/BbWPNt8s543Po3SN8Sp59iVkc/iFfvtr5+Z3JvYSH+nnkMQhNZNhH8DVf/+Gda8IyhDOqIf/pDTg/9kXrl9NM7RA9pz89AYPERvHkEQnEyEfz3JkhXjt3OQik+jDIzEc/wLTh+350xBBXOX7sTXU8uC+0T7viAIriOGfKwn8/51SMWnAdCPmen04DeZrbzx1V5kGWbcmiCCXxAElxJX/vUgleRQs/MbVO0T0I/6Owq1h9PP8ePvWZRUmJgwLIYO4b5OP74gCMKfiSv/eqje+jlYzegGT3VJ8BeWVPHLjpN0i/Yn8epopx9fEAThr+oV/pmZmUyaNInRo0czadIksrKyam2TkpLChAkT6NGjB0lJSQ7r3nvvPW688UbGjRvHhAkTWtT0jlKFAeup/Wi6XuP0m7vnvP7VXixWmfvGxrnk+IIgCH9Vr2afOXPmMHnyZMaPH8/KlSuZPXs2y5Ytc9imffv2zJ8/n9WrV2MymRzWJSQkMG3aNPR6PRkZGUydOpWUlBR0Op3z3omL2IZoltHEXeeS4+84mEd+cRXeeg2BbZr/5yEIwpWhzit/g8FAeno6iYmJACQmJpKenk5RUZHDdlFRUcTFxaFW1/57MnToUPR62w3M2NhYZFmmpKTEGfW7lFRZjGnvKlRhXVAFRzv9+Ot3n+aDlbZpF2dN6eP04wuCIFxMneGfk5NDaGgoKpWtr7lKpSIkJIScnJxGnXDFihVERkYSFhbWqP3dqXrzEsA1E7McOlHEZ2sPA/Diff1pK2bfEgTBjdza22fHjh288847fPzxxw3e91Kz0NclONinwftYjWWUn0pFExBOeIJzJ2FfvvogX609jL+PBx/MGoGnTuPU41+uxnxe7iDqarjmWpuoq2FcUVed4R8eHk5eXh5WqxWVSoXVaiU/P5/w8PAGnWjPnj089dRTLF68mJiYmAYXajBUIElyg/cLDvahoKC8wftVrbdNzqIZNr1R+1/MjoN5fHX2iv+hm7pTWV5NZXm1045/uRr7ebmaqKvhmmttoq6GaWxdSqXikhfNdTb7BAYGEhcXR3JyMgDJycnExcUREBBQ7yJSU1OZOXMmCxcupHv37vXer6mYj+/EcmwbCp8glMEdnHbcrNwyexv/u/+8ji7t/Zx2bEEQhIZQyLJc5+X0sWPHmDVrFmVlZfj6+pKUlERMTAzTp09nxowZxMfHs2vXLp588kkqKiqQZRkfHx/mz5/P0KFDmThxImfOnCE09PxkJwsWLCA2Nrbehbrryl+qLMb47Wzk6nK8Jr+B0juwwee8EFmWeWX5bo6dKePF+/rTt0fbK+oqw9VEXQ0XHOxDbm4xxcUFWCymundwE6VSiSRJTV1GLS21LrVai79/MCqVY0NOXVf+9Qr/5sAd4S/LMsbv5iAZTqIb/jCaToMafL4LMZRW86/PdlNUVsPU67swvE+7Zhsaoq6Gaa51ga22gwcPo9N54uXl6/QhSRpLrVZisTS/kG2JdcmyTGVlGdXVRoKCHJviL7vZpzWxntiLZDiJtv9EpwW/LMu8ejb4r+sTwXW9I5xyXEGoD4vF1KyCX3AuhUKBl5dvo77ZibF9/qRm748ovALQxo922jF/2nYCw9ngv+v6+jdzCYKziOC/sjX25yvC/yxr0Rmk/ON4DLoThVp72cfLMVTy1YajpB4zEOqv584RnZ1QpSAIgnOI8D/LnLYalCrUXa6+7GOdKazk//6zHYA+XYK574auqFWihU0Qbr11HB4eHqjVGiwWM3fcMZVx426+5D733juZDz/8GA+PSw9/kpa2jwUL5qNWq3nssSfp08e5z+dcaUT4AzW7vsN8aAua+NEodZf3MEVZpcke/H+/pQd9Y10zGJwgtFSvvLKAqKgYjh8/yrRpU7nqqsEEBQVfdPulSz+v13F/+eUnxo5NZPLkuxtUj8ViueCwNFe61veO/8JadArTHltbv8eA2y77eIu/TwNg8sjOIvgF4RJiYjrh4+NLQUE+RmMl77zzJqWlJZjNZm6//U5uvPEmAIYM6ceaNZvx9PTk1lvHMWbMjezcuR2DoZA775zKxImT+PzzZaxfvxadTseaNav58MOPOX78GG+//TrV1VXodHqeeOKfxMV1JycnmwceuIuxY8fxxx87uemmWxg8eBjvvPM6p06dBGDkyNHcddd9VFZWsGjRWxw7dgSTyUTv3v147LGZ9uFuWrJWHf5SVRnGb/4PAM9b5qBQXd7HcfBEMYdPl5J4dRQj+7V3RomC4DS/peWQktq4MbnqMiQhnMHxDXvqPzV1L23a+NGpUxceeug+5sx5maioaIzGSu6//y569EggKiq61n7V1dV8+OEScnKyufvuSYwdO47Jk+8mM/M4XbvGMXHiJMxmM88//zTPPTeHfv0GsHPndp5//mm++moFAKWlpcTFdePRR58A4LHHHmLw4CG8/PICAPvAk4sWvUWvXn2YNev/kCSJuXNfYNWqH7jpplsu49NqHlp1+Nek2Ial9rh6CkrPNpd1LFmW+WLdYdp4a0m8KtoJ1QnClem5555GkiTOnDnNvHmvcvr0KU6cyGTOnOfsiBd15wAAFyBJREFU25jNZrKyMi8Y/iNHXg9AeHhb+zeHv2538uQJNBoN/foNAKB//4FoNBpOnjyBp6cnWq0Hw4ePAsBoNLJ/fyqLFr1v39/Pz/b0fUrKZg4ePMCXX34G2P7whISEciVoteFvyc7AkrkLbd+b0fYYddnH+0/yQU4XVDJlVBe0mpb/lVC48gyOb/jVuSuca/PfsGEdr7wyl1dffZM2bfzq3bav1Z7vjadUKrFaLQ2uQa/X1bOLpMwrr7xORES7Bp+juWuVXVBkyUrV2kUodD5oE8Zc9vEyc8rYeiCXXp2CxENcglBPw4ePpH//QWzcuA6dTsfq1avs606cyKKysqLRx46MjMJsNvPHH7sA2L17JxaLhcjIqFrbenp60qNHgv3qHs43+wwePIzlyz/BarXal2dnn2l0Xc1Jq7zyN6f9AjWVaAdPRaG5vNmzMnPKmPfJLvQeaqaP64ZSKR6oEYT6evjhR7n//qksWPA2y5cv5YsvPsVqlQgICOCll15t9HE1Gg3z5y9wuOH78stJaDQXHj599ux5vPXWAlat+hGlUsWoUaOZOvVeHn/8HyxevJB7770ThUKBRqNlxox/0LZty7/Ia3Vj+8gWE5Wf/wOlf1v0ic+gUDT+y8/R06Ukff4HVknm3rFdGdazbaPrai5EXQ3TXOsCW21pafsJC6t9tduUWuIYOk2pPnXl5p6o9XOua2yfVnflb81OR64uR9vzhssKfoCVKcfRaVXMnTaAAF8x/64gCC1Hq2rzl0pzqVr9Nnh4oQq/vHF29hwu4EBWMTdeFS2CXxCEFqfVXPmbj/xO9caPANAPf/iy2vqLyqr54IcDtAv2YmS/K68XgCAIV74rPvytVRVUfvciUmEWAJ4TX0IVGHlZx1y17QRmi8S0G+PEmD2CILRIV3T4y7JM9qezkQpPgkqD58S5qPzqf1P2QkoqatiyL4dhPdsSHebrpEoFQRDcq16XrZmZmUyaNInRo0czadIksrKyam2TkpLChAkT6NGjB0lJSQ7rrFbr/7d371FR1/kfx5/MIAgKy51G4ejxAk4qWdjShfDEkiDiYrq4LuUFE808smeLLTAFJbwg2WbGpmlq3rL8mZpEYKy/0u1nqWFeorwFIjCC3FQwlZn5/v4gJ3G4zTAMyHwe53ROM9/ba77fj29mPvOdz4fFixcTEhLCM888w86dO00SvjXaKwXUXymix9AQHF5Y1+7CD7Al5wxarUT4413rDgpBEARDtKn4JycnEx0dTU5ODtHR0SQlJemt4+3tzZIlS3jhhRf0lu3bt4+ioiL279/Pxx9/zOrVqykuLm5/+lZIt+ro4doH20cnmmR/Jy9UcPxcBQ8NcsXDyc4k+xQESxIYOJIbN240em7s2D+hUpW2um18fBwlJQ1149KlImJioomJiWb//i86JGt312rxr6ysJD8/n4iICAAiIiLIz8+nqqqq0Xr9+vVDqVQ2OTRqVlYWUVFRyGQyXFxcCAkJITs720QvoXnW3sPxmv0OVjbtL9RXan7l3U9P09e9F9PGDDFBOkEQDPHmm+/ohln4+usDDBv2EBs3bmf06DFt3odabfhQEN1Vq33+KpUKT09P3RCmcrkcDw8PVCoVLi4ubTqISqWiT5/fu1wUCgWXL182MrJhTDGFXcOgbeewsoKXJ43A0b79M30JgqCvuSGb7yxbseJfnD9/jk8++QitVsupUydYsmQFkiSRnr6Umppq5HI5s2bN5bHHGiZmCgwcSUxMLIcPf0NAwOOUl5fRo0cPiosvUVJSzKhRT/Pkk0Fs2LCWsrIyJk2KZtKkv3XmaTCL++YL35Z+qdYad/f2TdDyVV4xP5yvYMoYJT4D3Nq1r7u1N1dHEbkM01VzQcPAZ9bWDR/wb/38X27/fLBDjmMzJAjbIYFtXv9OpgZWyOW/57x9+xYffPAhpaWlPPdcFOPGRWJvbw+AXC4jPHwsJSWX+PXXX4mL+wcAM2ZMZfz4Cfz5z+MpKPiFF1+cyY4du3B2dgbAzs6OTZu2ApCSkkxhYQHvvrsGrVbDs89GcONGHe+9t56Kigr++tdnGT/+Wd0xu4LG50ufTCYzuB22WvwVCgVlZWVoNBrkcjkajYby8nIUiraPDqhQKCgtLcXPzw/Q/yTQFqYa3sFQPxVWseHzn+jr3otRfg+Y7Kf8XXVYAJHLMF01FzRk02q1uqEBtFqJjhrNRauVDBoaofG6UqPtg4OfQa3W4uHxAA4OjqhUl3VDNms0Da9Hq/19mxs36jh37gxhYRGo1Vq8vfszaJAPJ06cIDAwCIDQ0HDd/iVJIjBwFDKZNTKZNd7e/QgIeOK3bmk3vWN2trYM76DVavXaYbuHd3B1dUWpVJKZmUlkZCSZmZkolco2d/kAhIWFsXPnTkaPHk1NTQ25ubls27at9Q072YWSq6Tv+AFruRVTQn2RmaALSRA6Sw+fJ+nh82SnZnBycubq1Rrc3R8AGvrga2trcXJy1q1jiiGb72Vn1/hdvK1t42PY2Nia/JhdXZvu9lm0aBFbt24lNDSUrVu3snjxYgBiY2M5daph2sJjx44RFBTExo0b2bFjB0FBQRw6dAiAyMhIvLy8GD16NJMmTWLu3Ll4e3ftma7UGi1LtnyPrY2cN+c+yYhBpuvuEQRL9eijAezZs0v3+LPPdjN06HB69jT+F/f29r0YNMiHL77IBKCwsIALF84ydOjwduftztrU5z9w4MAm781ft26d7v9HjhzJwYNN9yfK5XLdH4z7wa16DXNWfg3AhKAB4gteQTCRuLhXWL16JdOmTcbKSoaHhycLF6a0e7/Jyamkpy/lk0+2I5fLWbAgRdffLzTN4oZ0bo1ao+Vfn5zgp4vVKPs5Ez95hEnuGGpvLnMRuQzTVXOBGNLZUPdzLjGkswlkHb7ITxereexBT2LHPdghhV8QBKGziVHJ7nLtxm32/LcAmx4yUfgFQejWRPH/jSRJJG84AsCLkcNE4RcEoVsTxf83+YXVXK29zZPDHxB39giC0O2J4v+bnKNF9LbrQXSIT2dHEQRB6HCi+APfnFJx+pcqnhnphZ2t+A5cEITuz6IrnSRJbPriZw6dVAHw9CNiSkZBECyDRb/zP3G+Ulf44yePoLddj05OJAiCYB4W+87//06rWJ/5EwCr4gJxEL/iFQTBgljkO/+rdbd1hT9uop8o/IJgRuPHjyUyMhSNRqN7LitrH4GBI9m162MApk+P5tatm63u64MP1lJfX99hWTtCUdFFZs+OYfLkCcyeHcOlS0VNrvfNN4eIiYlmypTJTJv2N77++oBJc1hk8d9/pOFkTwvzZcRgcVunIJibq6s7R44c1j3OytqHj8/vM+Rt2rQdW9vWB3vbuHGdUcW/M2f0evPNZUyYEMWOHZ8yYUIU6elL9daRJIk33khi4cIUtmzZwcKFKaSmLkKrNd3wExZV/DVaLe9+eoovvivCb6Aro0b07exIgmCRwsMjyMpqGIWzpKSYmzdvMnDgIN3yO3P9XrxYyIQJY7l8ueG7uQ0b3ic5ORGAlSvTAJgzZwbTp0dz7twZxo79k24fKlVpo8eBgSP54IO1zJw5lY0b1/Hjj6eZN282M2Y8z4wZz/PNN4eazXvu3BnmzJnBtGmTmT49mszMvUa97urqKs6e/ZmQkFAAQkJCOXv2Z6qrq/XWlclk1NbWAlBbex1XVzdkMtOVbIvq8//fvBLyzl4BYOKogZ2cRhDM6zvV9xxWHe2QfT+ueJQAhX+b13/44ZHs3v0/XLt2jezszwkLC+fMmZ/11uvXrz+zZr1EUlIiM2e+yJdfZrN+/WYAXnnlNXbv3sl7723A3t6+TZPA29rasn79Zq5fv05c3GzS09/Bzc2NiooKYmOnsnnzxzg46M+ItWrVSmJiZvHHPz7W5H4XLHiV4uLiJpetXbtB9ymmrKwMNzePRtPiurm5U15e1mgUUisrK1JSlpGY+Ap2dnbU1d0gPX1Vq6/PEBZT/H8srGJ77jn6PeBA3EQ/nB1sW99IEIQOYWXVMGPXf/6zn9zcHNas2dBk8QcICxvL998fJTHxFTIy1tOrl/FTuo4ZEwHA6dMnUKlKiY+PuyuTFSUllxgy5EG97UaNCub111+lT5++vPfeeuztezVanpq6wuhMTVGr1WzZsolly1byyCOPkJeXR1JSAlu37jTZ9JIWUfwvXr7O2r0/AjAjXCkKv2CRAhT+Br0772hhYWOZPXs6Dz30MH/4g1Oz69XX11NQ8Au9eztQXV3Z7HpyubzRsO+3b9/WW+fOjF6SBAMHDiYj4/c5SZobOvmXX85z8uQP7N2b3Wzhbes7f09PTyoqyhtNi1tRcQUPD89G25w/f5bKyiv4+Y0AwM9vBHZ2dly8WIBSObTZc2AIiyj+uw/9Qu2v9SyYOhJvD+PfNQiCYDp9+3oRG/sSDz7YcjHLyFiFr+8Q5s9PJj4+jjVrNuiKpb19L+rqarG3t8fFxRW1Wk1x8SW8vLz58svsZvc5bJgfxcVF5OUd45FHRgKQn/8jgwcP0RvU8cCBXNzd3XWF/9q1azg6OjZap63v/J2dXRg0yIfc3BxCQ8PJzc1h8GBfvYln3N09KC8vp6iokAEDBlBYWEBVVRV9+5ruh6htKv4FBQUkJCRQU1ODk5MTaWlp9O/fv9E6Go2G1NRUDh06hJWVFbNmzSIqKgqAyspKEhMTUalUqNVqAgICWLBgAdbWHf+35/iZck5eqCT8sX4M6OPY+gaCIJhNZOSEFpcfPPgVx49/z/vvb8LW1pYZM2JZtOh13nlnDdbW1kye/BxxcS9ia9uT1avX8ve/v8I//jEXJycnHn88sNn9Ojo6snz5W2RkrGLVqpWo1fX07evF8uVv6RX/4OBnWLZsMc899xfs7Ozx93+UOXPmGf2a//nP+aSmJrNx43ocHBxYuPD3WQ7j4+OYOfNFhgx5kPj4BBYseA2ZTIYkQWJiEo6OfzD6uPdq00xeU6dOZeLEiURGRrJ371527drF5s2bG62zZ88e9u3bx7p166ipqWH8+PFs374dLy8vlixZgrW1Na+99hr19fVER0cTExNDeHh4m4MaM5NX3c16lm7Jo+5mPUtiA+jVs+v8grerzgAlchmmq+YCMZOXoe7nXMbM5NXqfUOVlZXk5+cTEdHwRUlERAT5+flUVVU1Wi8rK4uoqChkMhkuLi6EhISQnd3wscvKyoq6ujq0Wi23b9+mvr4eT09PvWOZ2tmiGiqv/srcZ4d1qcIvCILQ2Vot/iqVCk9Pz0a3Jnl4eKBSqfTW69Onj+6xQqHg8uXLALz00ksUFBQQGBio+8/fv+O/eBox2I0ti8MY7NX8l0mCIAiWyCxf+GZnZ+Pr68uHH35IXV0dsbGxZGdnExYW1uZ9tPTxpTX2XfRdv7u7/v3EXYHIZZiumgsafihkbd31fsvZFTPB/ZtLJpMZ3A5bLf4KhYKysrJGtyaVl5ejUCj01istLcXPzw9o/Elg69atLF26FJlMhoODA8HBwXz33XcGFX9j+vyh6/bJilyGEbkM5+7ugFarpb5e06WmJb2f+9Y7Q2u5JElCq9XqtcN29/m7urqiVCrJzGz4KXZmZiZKpRIXF5dG64WFhbFz5060Wi1VVVXk5uYSGtrwE2YvLy8OHjwINNx7e/jwYQYPHtzaoQVBaCdraxvq6q7Rhvs6hPuQJEnU1V3D2trwwSnb1O2zaNEiEhIS+Pe//42joyNpaQ1jasTGxhIXF8fw4cOJjIzkxIkTjB49GoC5c+fi7e0NwPz580lOTmbcuHFoNBoCAgKYNGmSwWEFQTCMs7M71dVXqK2t6ewoOjKZzKQDlJnK/ZrL2toGZ2d3g/fbpls9uwLR7WMeIpdhumou6LrZRC7DGJur3d0+giAIQvcjir8gCIIFum/G9pHJjL9boT3bdiSRyzAil+G6ajaRyzDG5Gptm/umz18QBEEwHdHtIwiCYIFE8RcEQbBAovgLgiBYIFH8BUEQLJAo/oIgCBZIFH9BEAQLJIq/IAiCBRLFXxAEwQKJ4i8IgmCB7pvhHYxRUFBAQkICNTU1ODk5kZaWRv/+/Tv8uNXV1bz66qsUFRVhY2NDv379SElJwcXFBV9fX3x8fJDJGv7urlixAl9fXwAOHDjAihUr0Gg0DB06lGXLlmFnZ2fSbMHBwdjY2GBrawtAfHw8Tz31FD/88ANJSUncunWLvn37kp6ejqurK0CLy0yhuLiYuXPn6h5fv36d2tpajhw50mzejsqVlpZGTk4OJSUl7Nu3Dx8fH6DltmTssvbmaqmdAWZpa82dL2Ovm6muaVO5Wmpn7clsiJaumbHnxehsUjc2ZcoUac+ePZIkSdKePXukKVOmmOW41dXV0rfffqt7vHz5cikxMVGSJEny8fGRamtr9bapra2VnnjiCamgoECSJEmaP3++tHr1apNne/rpp6UzZ840ek6j0UghISHS0aNHJUmSpIyMDCkhIaHVZR0lNTVVWrx4cbN5OzLX0aNHpdLSUr3jttSWjF3W3lwttTNJMk9ba+58GXPdTHlNm8t1t7vbmbGZDdXcNTP2vLQnW7ct/hUVFZK/v7+kVqslSZIktVot+fv7S5WVlWbPkp2dLU2bNk2SpOb/QWZlZUmzZs3SPT558qQUHh5u8ixNNfATJ05IY8eO1T2urKyURowY0eqyjnDr1i0pICBAOn36dLN5zZHr7uO21JaMXWaKXPe6u51JknnbWluLv7nbWnM57m1nxmZurzvXzNjz0p5s3bbbR6VS4enpiVwuB0Aul+Ph4YFKpdKbgrIjabVaPvroI4KDg3XPTZkyBY1GQ1BQEPPmzcPGxqbRnMcAffr0QaVSdUim+Ph4JEnC39+fl19+We/YLi4uaLVaampqWlzm5ORk8mwHDhzA09OToUOHNpvX0dHRrLlaakuSJBm1zNRtsKl2Bp3b1gy9bua8pk21M2MytyfX3dfM2PPSnmziC98O9sYbb2Bvb8/zzz8PwFdffcWnn37Ktm3bOH/+PBkZGWbNs23bNj777DN27dqFJEmkpKSY9fit2bVrFxMnTtQ97up5u4p72xl0blvr6tft3nYG5s/c1DUzp25b/BUKBWVlZWg0GgA0Gg3l5eUoFAqzZUhLS+PixYu8/fbbui/d7hy/d+/eREVFkZeXp3u+tLRUt21paWmHZL2zTxsbG6Kjo8nLy9M7dlVVFTKZDCcnpxaXmVpZWRlHjx5l3LhxLea987y5crXUloxdZkpNtbM7uaFz2pox181c17SpdmZsZmPde82MPS/tydZti7+rqytKpZLMzEwAMjMzUSqVZuvyeeuttzh9+jQZGRnY2NgAcPXqVW7evAmAWq0mJycHpVIJwFNPPcWpU6coLCwEYMeOHYwZM8akmW7cuMH16w1zgUqSRFZWFkqlkmHDhnHz5k2OHTumO3ZYWBhAi8tMbffu3YwaNQpnZ+cW85o7V0ttydhlptJUO4PObWvGXjdzXdN721l7MhujqWtm7HlpT7ZuPZnLhQsXSEhI4Nq1azg6OpKWlsaAAQM6/Ljnzp0jIiKC/v3707NnTwC8vLyYOXMmSUlJWFlZoVarefjhh5k/fz69evUCIDc3l/T0dLRaLUqlkuXLl2Nvb2+yXJcuXWLevHloNBq0Wi0DBw5kwYIFeHh4kJeXR3JycqPbxdzc3ABaXGZKoaGhvP766wQFBbWat6Nypaamsn//fioqKnB2dsbJyYnPP/+8xbZk7LL25nr77bebbGcZGRkcP37cLG2tqVxr1qwx+rqZ6po2dx1Bv52B+dpac7UhIyPD6PNibLZuXfwFQRCEpnXbbh9BEASheaL4C4IgWCBR/AVBECyQKP6CIAgWSBR/QRAECySKvyAIggUSxV8QBMECieIvCIJggf4fPLsqDa12XOwAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B5yJ1-OuIhL3"
      },
      "source": [
        "To compare the performance of all the methods, one needs to run extensive experiments on larger portions of the datasets for longer epochs. This can't be achieved in a tutorial so we invite the most curious to dive into the paper : [Fast Offline Policy Optimization for Large Scale Recommendation](https://arxiv.org/abs/2208.05327)"
      ],
      "id": "B5yJ1-OuIhL3"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ES2kWZ7CJSto"
      },
      "outputs": [],
      "source": [
        ""
      ],
      "id": "ES2kWZ7CJSto"
    }
  ],
  "metadata": {
    "colab": {
      "name": "Reward_Optimizing_Slate_Recommendation_with_DL_and_MIPS_Part_2.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}