BPR_OPT_Binary_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sy\n",
    "import pandas as pd\n",
    "\n",
    "import time\n",
    "import sys\n",
    "import os\n",
    "import ast\n",
    "import json\n",
    "import copy \n",
    "import itertools\n",
    "\n",
    "from itertools import islice\n",
    "from tqdm import *\n",
    "from math import ceil\n",
    "from collections import defaultdict\n",
    "from collections import Counter\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score\n",
    "from scipy.sparse import csr_matrix, dok_matrix\n",
    "from ipynb.fs.full.Random_Sample_Mapper import *\n",
    "\n",
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read In Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "notebook_path = os.path.abspath(\"BPR_OPT_Binary_Model.ipynb\")\n",
    "users_items_file_path = os.path.join(os.path.dirname(notebook_path), \"data/australian_users_items.json\")\n",
    "items_file_path = os.path.join(os.path.dirname(notebook_path), \"data/items_meta_data.json\")\n",
    "users_meta_data_file_path = os.path.join(os.path.dirname(notebook_path), \"data/users_meta_data.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_items = []\n",
    "with open(users_items_file_path, 'r') as data:\n",
    "    for line in data:\n",
    "        users_items.append(ast.literal_eval(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(items_file_path, 'r') as data:\n",
    "    games_dict = json.load(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(users_meta_data_file_path, 'r') as file:\n",
    "    users_meta_data = json.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process Data into Training and Testing Sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using default dict for efficient data retrieval for users-items playtime relationship"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "usersPerItem = defaultdict(set)\n",
    "itemsPerUser = defaultdict(set)\n",
    "playtimesPerItem = defaultdict(dict)\n",
    "playtimesPerUser = defaultdict(dict)\n",
    "itemNames = defaultdict(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for game in games_dict:\n",
    "    if 'owners' in games_dict[game]:\n",
    "        usersPerItem[game] = set(games_dict[game]['owners'].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for user in users_items:\n",
    "    u_id = user['user_id']\n",
    "    items = [item['item_id'] for item in user['items']]\n",
    "    itemsPerUser[u_id] = items\n",
    "    playtimesPerUser[user['user_id']] = dict((item['item_id'], item['playtime_forever']) for item in user['items'])\n",
    "    for item in user['items']:\n",
    "        itemNames[item['item_id']] = item['item_name']\n",
    "        playtimesPerItem[item['item_id']][user['user_id']] = item['playtime_forever']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scheduled Sampling with Map-Reduce Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "nUsers = len(itemsPerUser)\n",
    "nItems = len(usersPerItem)\n",
    "users = list(itemsPerUser.keys())\n",
    "items = list(usersPerItem.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_item_counts = dict((k, len(v)) for k, v in itemsPerUser.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "datafile = 'data/sample_in.tsv'\n",
    "mapout1 = 'data/sample_map1.tsv'\n",
    "mapout2 = 'data/sample_map2.tsv'\n",
    "outfile = 'data/sample_out.tsv'\n",
    "\n",
    "f = open(datafile,'w')\n",
    "for u, its in itemsPerUser.items():\n",
    "    for i in its:\n",
    "        print(default_formatter(u,i), file=f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run two stages of mapreduce\n",
    "mapper = Mapper(user_item_counts)\n",
    "mapreduce(datafile, mapout1, mapper=mapper, reducer=reducer)\n",
    "mapreduce(datafile, mapout2, mapper=indicator_mapper)  # map the data again\n",
    "mapreduce([mapout1, mapout2], outfile, reducer=indicator_reducer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trim(u, i, j):\n",
    "    return u[1:len(u)-1], i[2:len(i)-2], j[1:len(j)-2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data(filepath):\n",
    "    f = open(filepath)\n",
    "    samples = [map(str, line.strip().split()) for line in f]\n",
    "    return [trim(u, i, j) for u, i, j in samples]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_random_batches(data, batch_size=1024):\n",
    "    batches = []\n",
    "    random.shuffle(data)\n",
    "    num_batches = int(len(data)/batch_size)\n",
    "    for i in range(num_batches):\n",
    "        mini_batch = data[i*batch_size:(i+1)*batch_size]\n",
    "        batches.append(mini_batch)\n",
    "    return batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = create_data(outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arnold/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "sample_data = train_test_split(data, train_size=0.05)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleUsersPerItem = defaultdict(set)\n",
    "sampleItemsPerUser = defaultdict(set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "for u, i, j in sample_data:\n",
    "    sampleUsersPerItem[i].add(u)\n",
    "    sampleItemsPerUser[u].add(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = sample_data\n",
    "sampleItems = list(sampleUsersPerItem.keys())\n",
    "sampleUsers = list(sampleItemsPerUser.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = []\n",
    "for u in sampleItemsPerUser:\n",
    "    for i in sampleItemsPerUser[u]:\n",
    "        j = random.choice(sampleItems)\n",
    "        test_data.append((u, i, j))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utility Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inner(x, y):\n",
    "    return sum([a*b for a,b in zip(x,y)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def graph(data, title):\n",
    "    plt.plot([x for x in range(len(data))], data, linewidth=2.0)\n",
    "    plt.title(title)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_label(u, i, j):\n",
    "    c = Counter(itemsPerUser[u])\n",
    "    if c[i] >= c[j]:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_outputs(sample, binary):\n",
    "    predictions = []\n",
    "    labels = []\n",
    "    for u, i, j in sample:\n",
    "        predict = sigmoid(prediction(u, i, j))\n",
    "        label = binary(u, i, j)\n",
    "        predictions.append(predict)\n",
    "        labels.append(label)\n",
    "            \n",
    "    return np.rint(predictions), labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "    differences = [1 if x == y else 0 for x, y in zip(predictions, labels)]\n",
    "    return sum(differences) / len(differences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = [binary_label(u, i, j) for u, i, j in train_data]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sigmoid Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\sigma(x) = \\frac{1}{1 + e^{-x}}\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    #Numerically stable sigmoid function.\n",
    "    #Taken from: https://timvieira.github.io/blog/post/2014/02/11/exp-normalize-trick/\n",
    "    if x >= 0:\n",
    "        z = np.exp(-x)\n",
    "        return 1 / (1 + z)\n",
    "    else:\n",
    "        # if x is less than zero then z will be small, denom can't be\n",
    "        # zero because it's 1+z.\n",
    "        z = np.exp(x)\n",
    "        return z / (1 + z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple (Bias Only) Latent Factor Model with Binary Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_history = []\n",
    "train_accuracy_history = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "itemBiases = defaultdict(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unpack(theta):\n",
    "    global itemBiases\n",
    "    itemBiases = dict(zip(items, theta))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "f(i, j) = \\beta_i - \\beta_j\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "p(i >_u j) = \\sigma(f(i, j))\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prediction(u, item_i, item_j):\n",
    "    return itemBiases[item_i] - itemBiases[item_j]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\text{Cost Function (arg min)}:= \\sum_{u,i,j} -\\ln(\\sigma(\\beta_i - \\beta_j)) + \\lambda \\sum_i \\beta_i^2 = \\sum_{u,i,j} -ln\\left( \\frac{1}{1 + e^{\\beta_j - \\beta_i}} \\right) + \\lambda \\sum_i \\beta_i^2\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(theta, lamb, train_data, train_labels):\n",
    "    unpack(theta)\n",
    "    cost = 0.0\n",
    "    predictions = []\n",
    "    for u, i, j in train_data:\n",
    "        x = prediction(u, i, j)\n",
    "        predictions.append(sigmoid(x))\n",
    "        cost += np.log(sigmoid(x))\n",
    "\n",
    "    for i in itemBiases:\n",
    "        cost -= lamb*itemBiases[i]**2\n",
    "\n",
    "    train_accuracy = accuracy(np.rint(predictions), train_labels)\n",
    "    loss_history.append(-cost)\n",
    "    train_accuracy_history.append(train_accuracy)\n",
    "    #print('iteration {0} Cost: {1}'.format(len(loss_history), -cost))\n",
    "    #print('iteration {0} Training Accuracy: {1}'.format(len(train_accuracy_history), train_accuracy))\n",
    "    #print('-------------------------------------------------------------------')\n",
    "\n",
    "    return -cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial }{\\partial x} ln\\sigma(x) = \\frac{1}{1 + e^x} = \\sigma(-x)\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Partial Derivatives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial f}{\\partial \\beta_i} = -\\frac{e^{\\beta_j - \\beta_i}}{1 + e^{\\beta_j - \\beta_i}} + 2 \\lambda \\beta_j = -\\frac{1}{1 + e^{\\beta_i - \\beta_j}} + 2 \\lambda \\beta_j\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial f}{\\partial \\beta_j} = \\frac{e^{\\beta_j - \\beta_i}}{1 + e^{\\beta_j - \\beta_i}} + 2 \\lambda \\beta_j = \\frac{1}{1 + e^{\\beta_i - \\beta_j}} + 2 \\lambda \\beta_j\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Important]: switch the sign of all partial derivatives to compute your gradient ascent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def derivative(theta, lamb, train_data, train_labels):\n",
    "    unpack(theta)\n",
    "    dItemBiases = defaultdict(float)\n",
    "    for u, i, j in train_data:\n",
    "        x = prediction(u, i, j)\n",
    "        dbase = 1 / (1 + np.exp(x)) # negative gradient descent for maximizing\n",
    "        dItemBiases[i] += -dbase\n",
    "        dItemBiases[j] += dbase\n",
    "    for i in itemBiases:\n",
    "        dItemBiases[i] += 2*lamb*itemBiases[i]\n",
    "    dtheta = [dItemBiases[i] for i in items]\n",
    "    return np.array(dtheta)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "res, f, d = sy.optimize.fmin_l_bfgs_b(cost, [0.0]*nItems, derivative, args=[0.001, train_data, train_labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unpack(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot([x for x in range(len(loss_history))], loss_history, linewidth=2.0)\n",
    "plt.title('Bias Only Loss History')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot([x for x in range(len(train_accuracy_history))], train_accuracy_history, linewidth=2.0)\n",
    "plt.title('Bias Only Training Accuracy History')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions, labels = generate_outputs(test_data, binary_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy(predictions, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('./data/bias_only_itemBiases', itemBiases)\n",
    "np.save('./data/bias_only_loss', loss_history)\n",
    "np.save('./data/bias_only_train_accuracy', train_accuracy_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Complete Latent Factor Model with Binary Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code extends the example above to implement a complete latent factor model (i.e., including low-dimensional user and item terms)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_history = []\n",
    "train_accuracy_history = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "itemBiases = defaultdict(float)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each user and item we now have a low dimensional descriptor (representing that user's preferences, and that item's properties), of dimension K."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "userGamma = {}\n",
    "itemGamma = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "for u in itemsPerUser:\n",
    "    userGamma[u] = [random.random() * 0.1 - 0.05 for k in range(K)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in usersPerItem:\n",
    "    itemGamma[i] = [random.random() * 0.1 - 0.05 for k in range(K)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use another library in this example to perform gradient descent. This library requires that we pass it a \"flat\" parameter vector (theta) containing all of our parameters. This utility function just converts between a flat feature vector, and our model parameters, i.e., it \"unpacks\" theta into our offset and bias parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unpack(theta):\n",
    "    global itemBiases\n",
    "    global userGamma\n",
    "    global itemGamma\n",
    "    index = 0\n",
    "    itemBiases = dict(zip(items, theta[0:index + nItems]))\n",
    "    index += nItems\n",
    "    for u in users:\n",
    "        userGamma[u] = theta[index:index + K]\n",
    "        index += K\n",
    "    for i in items:\n",
    "        itemGamma[i] = theta[index:index + K]\n",
    "        index += K"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "f(u, i, j) = \\gamma_u \\gamma_i + \\beta_i - (\\gamma_u \\gamma_j + \\beta_j)\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "p(i >_u j) = \\sigma(f(u, i, j))\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prediction(user, item_i, item_j):\n",
    "    return inner(userGamma[user], itemGamma[item_i]) + itemBiases[item_i] - (inner(userGamma[user], itemGamma[item_j]) + itemBiases[item_j]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cost Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\text{Cost Function (arg min)}:= \\sum_{u,i,j} -\\ln(\\sigma(\\gamma_u \\gamma_i + \\beta_i - (\\gamma_u \\gamma_j + \\beta_j))) + \\lambda [\\sum_i \\beta_i^2 + \\sum_i ||\\gamma_i||_2^2 + \\sum_u ||\\gamma_u||_2^2 ]\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\sum_{u,i,j} \\ln(\\sigma(\\gamma_u \\gamma_i + \\beta_i - (\\gamma_u \\gamma_j + \\beta_j))) = \\sum_{u,i,j} ln\\left( \\frac{1}{1 + e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}} \\right)\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(theta, lamb, train_data, train_labels):\n",
    "    unpack(theta)\n",
    "    cost = 0.0\n",
    "    predictions = []\n",
    "    for u, i, j in train_data:\n",
    "        x = prediction(u, i, j)\n",
    "        predictions.append(sigmoid(x))\n",
    "        cost += np.log(sigmoid(x))\n",
    "        \n",
    "    for u in users:\n",
    "        for k in range(K):\n",
    "            cost -= lamb*userGamma[u][k]**2\n",
    "    for i in items:\n",
    "        cost -= lamb*itemBiases[i]**2\n",
    "        for k in range(K):\n",
    "            cost -= lamb*itemGamma[i][k]**2\n",
    "        \n",
    "    train_accuracy = accuracy(np.rint(predictions), train_labels)\n",
    "    loss_history.append(-cost)\n",
    "    train_accuracy_history.append(train_accuracy)\n",
    "    #print('iteration {0} Cost: {1}'.format(len(loss_history), -cost))\n",
    "    #print('iteration {0} Training Accuracy: {1}'.format(len(train_accuracy_history), train_accuracy))\n",
    "    #print('-------------------------------------------------------------------')\n",
    "        \n",
    "    return -cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Partial Derivatives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial f}{\\partial \\gamma_{u,k}} = \\frac{(\\gamma_{j,k} - \\gamma_{i,k}) \\cdot e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}}{1 + e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}} + 2 \\lambda \\gamma_{i,k} \n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial f}{\\partial \\gamma_{i,k}} = -\\frac{\\gamma_{u,k} \\cdot e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}}{1 + e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}} + 2 \\lambda \\gamma_{i,k}\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial f}{\\partial \\gamma_{j,k}} = \\frac{\\gamma_{u,k} \\cdot e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}}{1 + e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}} + 2 \\lambda \\gamma_{j,k}\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial f}{\\partial \\beta_i} = -\\frac{e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}}{1 + e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}} + 2 \\lambda \\beta_i\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "\\frac{\\partial f}{\\partial \\beta_j} = \\frac{e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}}{1 + e^{\\gamma_u \\gamma_j + \\beta_j - (\\gamma_u \\gamma_i + \\beta_i)}} + 2 \\lambda \\beta_j\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def derivative(theta, lamb, train_data, train_labels):\n",
    "    unpack(theta)\n",
    "    dItemBiases = defaultdict(float)\n",
    "    dUserGamma = {}\n",
    "    dItemGamma = {}\n",
    "    for u in users:\n",
    "        dUserGamma[u] = [0.0 for k in range(K)]\n",
    "    for i in items:\n",
    "        dItemGamma[i] = [0.0 for k in range(K)]\n",
    "    for u, i, j in train_data:\n",
    "        x = prediction(u, i ,j)\n",
    "        dbase = 1 / (1 + np.exp(x))\n",
    "        dItemBiases[i] += -dbase\n",
    "        dItemBiases[j] += dbase\n",
    "        for k in range(K):\n",
    "            dUserGamma[u][k] += (itemGamma[j][k] - itemGamma[i][k]) * dbase\n",
    "            dItemGamma_k = userGamma[u][k] * dbase\n",
    "            dItemGamma[i][k] += -dItemGamma_k\n",
    "            dItemGamma[j][k] += dItemGamma_k\n",
    "    for u in userGamma:\n",
    "        for k in range(K):\n",
    "            dUserGamma[u][k] += 2*lamb*userGamma[u][k]\n",
    "    for i in itemBiases:\n",
    "        dItemBiases[i] += 2*lamb*itemBiases[i]\n",
    "        for k in range(K):\n",
    "            dItemGamma[i][k] += 2*lamb*itemGamma[i][k]\n",
    "    dtheta = [dItemBiases[i] for i in items]\n",
    "    for u in users:\n",
    "        dtheta += dUserGamma[u]\n",
    "    for i in items:\n",
    "        dtheta += dItemGamma[i]\n",
    "    return np.array(dtheta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "complete_res, complete_f, complete_d = sy.optimize.fmin_l_bfgs_b(cost, \n",
    "                                [0.0]*nItems + # Initialize beta\n",
    "                                [random.random() * 0.1 - 0.05 for k in range(K*(nUsers + nItems))], # Gamma\n",
    "                                derivative, args=[0.001, train_data, train_labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "complete_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unpack(complete_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions, labels = generate_outputs(test_data, binary_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy(predictions, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot([x for x in range(len(loss_history))], loss_history, linewidth=2.0)\n",
    "plt.title('BPR Loss History')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot([x for x in range(len(train_accuracy_history))], train_accuracy_history, linewidth=2.0)\n",
    "plt.title('BPR Training Accuracy History')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('./data/complete_itemGamma', itemGamma)\n",
    "np.save('./data/complete_itemBiases', itemBiases)\n",
    "np.save('./data/complete_userGamma', userGamma)\n",
    "np.save('./data/complete_loss', loss_history)\n",
    "np.save('./data/complete_train_accuracy', train_accuracy_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_user(user):\n",
    "    \"\"\"\n",
    "    returns the predicted ratings for the specified user,\n",
    "    this is mainly used in computing evaluation metric\n",
    "    \"\"\"\n",
    "    user_pred = []\n",
    "    labels = []\n",
    "    copy_items = copy.deepcopy(items)\n",
    "    for i in itemsPerUser[user]:\n",
    "        copy_items.remove(i)\n",
    "        j = random.choice(copy_items)\n",
    "        predict = sigmoid(prediction(user, i, j))\n",
    "        copy_items.append(i)\n",
    "        user_pred.append(predict)\n",
    "        labels.append(binary_label(u, i, j))\n",
    "        \n",
    "    return np.array(labels), np.array(user_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auc_score():\n",
    "    score = 0.0\n",
    "    n_user = 0\n",
    "    for user in tqdm(users):\n",
    "        if len(itemsPerUser[user]) == 0: continue\n",
    "        y_true, y_pred = predict_user(user)\n",
    "        n_user += 1\n",
    "        if len(np.unique(y_true)):\n",
    "            score += accuracy_score(y_true, np.rint(y_pred))\n",
    "        else:\n",
    "            score += roc_auc_score(y_true, y_pred)\n",
    "    score /= n_user\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_score()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Map a Sample for Played and Purchased Items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "playedItemsPerUser = defaultdict(set)\n",
    "for u in itemsPerUser:\n",
    "    for i in itemsPerUser[u]:\n",
    "        if playtimesPerUser[u][i] > 0:\n",
    "            playedItemsPerUser[u].add(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "played_user_item_counts = dict((k, len(v)) for k, v in playedItemsPerUser.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "played_datafile = 'data/played_sample_in.tsv'\n",
    "played_mapout1 = 'data/played_sample_map1.tsv'\n",
    "played_mapout2 = 'data/played_sample_map2.tsv'\n",
    "played_outfile = 'data/played_sample_out.tsv'\n",
    "\n",
    "f = open(played_datafile,'w')\n",
    "for u, its in playedItemsPerUser.items():\n",
    "    for i in its:\n",
    "        print(default_formatter(u,i), file=f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run two stages of mapreduce\n",
    "played_mapper = Mapper(played_user_item_counts)\n",
    "mapreduce(played_datafile, played_mapout1, mapper=played_mapper, reducer=reducer)\n",
    "mapreduce(played_datafile, played_mapout2, mapper=indicator_mapper)  # map the data again\n",
    "mapreduce([played_mapout1, played_mapout2], played_outfile, reducer=indicator_reducer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "mixed_data = create_data(played_outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "played_data = []\n",
    "for u, i, j in mixed_data:\n",
    "    c = Counter(itemsPerUser[u])\n",
    "    if c[j] > 0:\n",
    "        played_data.append((u, i, j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "played_sample_data = train_test_split(played_data, train_size=0.05)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "playedSampleUsersPerItem = defaultdict(set)\n",
    "playedSampleItemsPerUser = defaultdict(set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "for u, i, j in played_sample_data:\n",
    "    playedSampleUsersPerItem[i].add(u)\n",
    "    playedSampleItemsPerUser[u].add(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "played_train_data = played_sample_data\n",
    "playedSampleItems = list(playedSampleUsersPerItem.keys())\n",
    "playedSampleUsers = list(playedSampleItemsPerUser.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "played_test_data = []\n",
    "for u in playedSampleItemsPerUser:\n",
    "    for i in playedSampleItemsPerUser[u]:\n",
    "        j = random.choice(itemsPerUser[u])\n",
    "        played_test_data.append((u, i, j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_played_label(u, i, j):\n",
    "    if playtimesPerUser[u][i] >= playtimesPerUser[u][j]:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "played_train_labels = [binary_played_label(u, i, j) for u, i, j in played_train_data]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the new data set on the two latent factor models "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "P(played | purchased)\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bias Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_history = []\n",
    "train_accuracy_history = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "played_res, played_f, played_d = sy.optimize.fmin_l_bfgs_b(cost, [0.0]*nItems, derivative, args=[0.001, played_train_data, played_train_labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'grad': array([ 0.00000000e+00, -1.09356244e-07,  0.00000000e+00, ...,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00]),\n",
       " 'task': b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH',\n",
       " 'funcalls': 212,\n",
       " 'nit': 205,\n",
       " 'warnflag': 0}"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "played_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "unpack(played_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAH9tJREFUeJzt3Xuc3HV97/HXe6+5koRkuSWBBAwoICCNENFjrVQIeAnn0VrhqETL4+Shgud4a4XSCtXyqLX2aLFKD5YIWIVyvJFaTjHlqLTVCMstEBGycsuSkGzMhVzIbnb3c/74fWd3dndmd7O32czv/Xw85jEz39935ved387Oe77f7+/3G0UEZmaWPzWVboCZmVWGA8DMLKccAGZmOeUAMDPLKQeAmVlOOQDMzHLKAZBjkv5e0p9Vuh39Sbpe0j+Ow/OGpFeN9fMOsc4PSPqPiVznWJms7w8bOw6AKibpOUmvSNoraaekf5G0sLA8Ij4UEZ8bh/WeKmmNpN2S9kj6saTzxno9Jda7KH3I702X5yRdPd7rHami9tZVYN23SvqLwdoz3PdH2s6/O15ttfHjAKh+74yIGcCxwFbgK+O5MkknAf8JPA4sBo4Dvg/8SNIbxnPdRWan13wZ8BlJyydovXaIKhF+1ssBkBMRcQD4DnBqoaz4W6CkOZJ+KKkt9RZ+KGlBUd0PSHomfaN/VtJ7y6zqeuDnEXFtROyIiD0RcSPwTeCv0nMVvmmulPSCpO2Sri31ZKnX8tF+ZeslXTKM1/xzYANweonnfbukRyS9LGmTpOuHu05Jr5a0VtIOSU9J+oOienNT7+dlSQ8AJw3VzlIkNUr6sqTN6fJlSY1p2bz099mV2vDvkmrSsk9LejH9nZ6SdP5I1p+eq/j9UXKdkr4JHA/8c+p1/XGq/y5JG1L9n0h6TdHzPpfauR7YJ+mPJH2337q/IunLI227DY8DICckTQPeA6wrU6UG+AZwAtk/9CvA36XHTgduBC6KiJnAecCjZZ7nbcD/KVF+F/DG1I6CNwGnAOeTfVN/TYnH3Qa8r+h1nAnMB+4ps/5CPUl6I3Aa8EiJKvuAy4HZwNuBDxeFStl1pm2xFvg2cBRZL+Nrkk5L1b8KHCDrcf1huozEtcAy4CzgTOAc4E/Tsk8CrUATcDTwJ0BIOgW4Cnh9+jtdCDw3wvX3V3KdEfF+4AVSTzMiviDpZOAO4GOp/j1kAdFQ9HyXkW332cA/AsslzYaeXsF7yL402DhyAFS/H0jaBbxM9uH816UqRcRvIuK7EbE/IvYANwC/XVSlGzhd0tSI2BIRG8qsbx6wpUT5FrL325yisj+PiFci4jHgMbIPuv7uBpZIWpLuvx/4p4joKLN+gO3ADuAfgKsj4r7+FSLiJxHxeER0R8R6sg+swusdbJ3vAJ6LiG9ERGdEPAx8F/h9SbXA7wGfiYh9EfEEWZiMxHuBz0bEtohoA/48tQPgIFnAnBARByPi3yM7qVcX0AicKqk+Ip6LiF8Pso5PpW/ou9J7ZP0gdcuts5T3AP8SEWsj4iDwRWAq2ReHghsjYlP6+28B7gfenZYtB7ZHxEODtMfGgAOg+l0SEbPJPhiuAn4q6Zj+lSRNk/S/JT0v6WWyf8jZkmojYh/ZP/WHgC1piOTVZda3neyDor9jyUJkZ1HZS0W39wMz+j8oItrJeg/vS8MclzH0N8N5ETEnIl6Thp8GkHSussnpNkm702ubN4x1ngCc2++D873AMWTfduuATUWren6ItpZzXL/HPp/KIAvxFrJ5lWeUJrojooXsW/f1wDZJd0o6jvK+GBGzCxfgjEHqllzncNoeEd1k22R+UZ1N/R5T3Ot6H/72PyEcADkREV0R8T2yb4lvKlHlk2TDMedGxBHAm1O50uPvjYi3kX2Q/wr4eplV/Ru93+SK/QHZ3MD+ETT/NrIP2fOB/Wlsf7S+DawBFkbELODvSa91iHVuAn5a/MGZhj4+DLQBncDCouc5foTt20wWNsXPsxkgzat8MiJOBN4JfKIw1h8R346IN6XHBmneZbQGW2daT9m2SxLZNnmx+Cn7PeYHwBmSTifrZX1rLNptg3MA5EQaE19BNgTzZIkqM8nG/XdJOhK4ruixR6dJvelAO7CXLEhK+XPgPEk3SDpS0sw0oXo58OmRtD19+HYDf8PYfTOcCeyIiAOSzgH+2zDX+UPgZEnvl1SfLq+X9JqI6AK+B1yfelSnAiuH0ZZGSVOKLjVkQ1J/KqlJ0jzgM2Rj5Uh6h6RXpQ/Wl8n+Fl2STpH01jRZfIDs71nu73RIyq0zLd4KnFhU/S7g7ZLOl1RP9uWiHfhZuecv2knh28ADEfHCWLTbBucAqH7/LGkv2T/tDcDKMuP3XyYbp91ONlH8r0XLasj+iTeTja3/NvCRUiuLiI1kPYwzySYgt5CNi18YEf85itdxO/Ba0ofgGPgI8FlJe8g+XO8azjrT/MgFwKVk2+Mlsm/ZjanKVWRDWS8Bt5JNrA9lL9mHdeHyVuAvgGaycfnHgYdTGcASsp7WXuDnwNci4iepDZ8n+xu+RDZJ/SfDWP9wlFsnwF+ShdUuSZ+KiKfIhnG+ktryTrJJ4sHmbSDrdb0WD/9MGPkHYexwIOlyYFUa3qjadeaZpOPJhhePiYiXK92ePHAPwCa9tOvoR4Cbq3mdeZaGvT4B3OkP/4njALBJTdKFZJOrW8nGh6tynXmW5pYKuylfN0R1G0MeAjIzyyn3AMzMcmpSn4hp3rx5sWjRoko3w8zssPLQQw9tj4imoepN6gBYtGgRzc3NlW6GmdlhRdKwjkAfcghI0mpJ2yQ9UVR2lqR1kh6V1JwOpCkcbHSjpBZlZ088u+gxKyVtTJfhHBxjZmbjaDhzALeSnZyp2BfITuR1FtlBNF9I5ReRHTCyBFgF3ARQdGTpuWRnNbxO0hzMzKxihgyAiLif7OjPPsXAEen2LNI5SoAVwO2RWUd2MrFjyU5LuzadH34n2el0/SMdZmYVNNI5gI8B90r6IlmIFE7zOp++Z/lrTWXlygeQtIqs98Dxx4/0PFpmZjaUke4G+mHg4xGxEPg4cEsqV4m6MUj5wMKImyNiaUQsbWoachLbzMxGaKQBsJLsrIeQ/frTOel2K31PhbuAbHioXLmZmVXISANgM72/nvRWYGO6vQa4PO0NtAzYnX7t517gAmW/OzuH7GyK946i3WZmNkpDzgFIugN4CzBPUivZ3jz/Hfjb9NudB0hj9mS//Xkx2S8H7Qc+CBAROyR9Dngw1ftsRPSfWB4z9z/dxtd+0sK5i+fy8bedPF6rMTM7rA0ZABFxWZlFv1WibgBXlnme1cDqQ2rdCB042MW6Z3bQUFc7EaszMzssVeW5gBbPmw7Ac9v3VbglZmaTV1UGwPFzpyFB6879dHR2V7o5ZmaTUlUGQGNdLfNnT6U7YNPOkfwGuZlZ9avKAIDeYaBn2zwMZGZWStUGwKK5aR7gNw4AM7NSqjcACj0ATwSbmZVUtQFw4jz3AMzMBlO1AbCoZ1dQTwKbmZVStQGwYM5UamvEi7te4cDBrko3x8xs0qnaAKivreHomY0AtO1pr3BrzMwmn6oNAIDpjdmZLvZ3uAdgZtZfLgJgX0dnhVtiZjb5VHkAZCeD29/uHoCZWX9VHQDTGtwDMDMrp6oDYHpD6gE4AMzMBqjqAJhWmAPwEJCZ2QBVHQDuAZiZlVfVAdAzB+AegJnZAFUdAD17AbkHYGY2QFUHQO9eQO4BmJn1V9UB0HscgHsAZmb9VXUAuAdgZlbekAEgabWkbZKe6Ff+UUlPSdog6QtF5ddIaknLLiwqX57KWiRdPbYvo7TpDYVzAbkHYGbWX90w6twK/B1we6FA0u8AK4AzIqJd0lGp/FTgUuA04Djg3ySdnB72VeBtQCvwoKQ1EfHLsXohpUxLQ0DeC8jMbKAhAyAi7pe0qF/xh4HPR0R7qrMtla8A7kzlz0pqAc5Jy1oi4hkASXemuuMaAO4BmJmVN9I5gJOB/yLpF5J+Kun1qXw+sKmoXmsqK1c+gKRVkpolNbe1tY2weZlpDe4BmJmVM9IAqAPmAMuAPwLukiRAJerGIOUDCyNujoilEbG0qalphM3L9P4egHsAZmb9DWcOoJRW4HsREcADkrqBeal8YVG9BcDmdLtc+bjp6QF4LyAzswFG2gP4AfBWgDTJ2wBsB9YAl0pqlLQYWAI8ADwILJG0WFID2UTxmtE2fiiNdTXU1oiOzm4OdnWP9+rMzA4rQ/YAJN0BvAWYJ6kVuA5YDaxOu4Z2ACtTb2CDpLvIJnc7gSsjois9z1XAvUAtsDoiNozD6+nfdqY11LLnQCf7O7qYNbWqD3swMzskw9kL6LIyi95Xpv4NwA0lyu8B7jmk1o2B6Q11KQA6mTW1fqJXb2Y2aVX9V2IfC2BmVlrVB4CPBTAzK63qA8DHApiZlVb1AeBjAczMSqv6APCxAGZmpVV9APTMAfg3AczM+qj6AOjZC8g9ADOzPqo+ANwDMDMrreoDoNAD2OtJYDOzPqo+AKbUZQHQftDnAjIzK1b1AdBYn73E9k4HgJlZsaoPgIba7CV2OADMzPqo/gCoK/QAvBeQmVmxqg+AxjQH4B6AmVlfOQiANATkH4QxM+uj6gOgZwjIewGZmfVR9QHgHoCZWWlVHwCFHoDnAMzM+spNAHgvIDOzvqo+ALwXkJlZaVUfAL09AAeAmVmx6g8AHwlsZlbSkAEgabWkbZKeKLHsU5JC0rx0X5JulNQiab2ks4vqrpS0MV1Wju3LKK9wLiAHgJlZX8PpAdwKLO9fKGkh8DbghaLii4Al6bIKuCnVPRK4DjgXOAe4TtKc0TR8uAo9AA8BmZn1NWQARMT9wI4Si74E/DEQRWUrgNsjsw6YLelY4EJgbUTsiIidwFpKhMp46BkC6uomIoaobWaWHyOaA5D0LuDFiHis36L5wKai+62prFx5qedeJalZUnNbW9tImtdHTY36hICZmWUOOQAkTQOuBT5TanGJshikfGBhxM0RsTQiljY1NR1q80rynkBmZgONpAdwErAYeEzSc8AC4GFJx5B9s19YVHcBsHmQ8gnho4HNzAY65ACIiMcj4qiIWBQRi8g+3M+OiJeANcDlaW+gZcDuiNgC3AtcIGlOmvy9IJVNiEYHgJnZAMPZDfQO4OfAKZJaJV0xSPV7gGeAFuDrwEcAImIH8DngwXT5bCqbEB4CMjMbqG6oChFx2RDLFxXdDuDKMvVWA6sPsX1jwgeDmZkNVPVHAoMPBjMzKyUXAdB7MJjPCGpmVpCLAPAZQc3MBspFAHgS2MxsIAeAmVlO5SIA/LvAZmYD5SIAenoABz0JbGZWkIsAcA/AzGygnASA9wIyM+svFwHgSWAzs4HyEQA+FYSZ2QC5CACfDdTMbKBcBEDvEJD3AjIzK8hVALgHYGbWKxcBUNgLyJPAZma9chEA7gGYmQ2UqwBo94FgZmY9chEAjT2ngnAAmJkV5CIAGnwqCDOzAXIRAI09B4J5N1Azs4J8BEC9TwVhZtZfLgKgodYngzMz62/IAJC0WtI2SU8Ulf21pF9JWi/p+5JmFy27RlKLpKckXVhUvjyVtUi6euxfSnneDdTMbKDh9ABuBZb3K1sLnB4RZwBPA9cASDoVuBQ4LT3ma5JqJdUCXwUuAk4FLkt1J0SjzwZqZjbAkAEQEfcDO/qV/SgiOtPddcCCdHsFcGdEtEfEs0ALcE66tETEMxHRAdyZ6k4I9wDMzAYaizmAPwT+b7o9H9hUtKw1lZUrH0DSKknNkprb2trGoHneDdTMrJRRBYCka4FO4FuFohLVYpDygYURN0fE0ohY2tTUNJrm9Wj0bwKbmQ1QN9IHSloJvAM4PyIKH+atwMKiaguAzel2ufJx5x6AmdlAI+oBSFoOfBp4V0TsL1q0BrhUUqOkxcAS4AHgQWCJpMWSGsgmiteMrunD11BbgwQHu4JOh4CZGTCMHoCkO4C3APMktQLXke310wislQSwLiI+FBEbJN0F/JJsaOjKiOhKz3MVcC9QC6yOiA3j8HrKvQam1teyv6OLA53dzKjNxeEPZmaDGjIAIuKyEsW3DFL/BuCGEuX3APccUuvG0LSGLAD2d3Qyo3HEI19mZlUjN1+Fp9RnRwO/0uGJYDMzyFEATGtIAeA9gczMgBwFwNSGbNhnv3sAZmZAngIgnRHUQ0BmZpncBMC01ANwAJiZZXITAFPTHMB+zwGYmQF5CoC0F9AB9wDMzIAcBUBhL6D9HZ1D1DQzy4fcBICHgMzM+spPAHgIyMysj9wEQO8QkAPAzAxyFAA9B4J5CMjMDMhTAHgIyMysj9wEgIeAzMz6yk0AeC8gM7O+8hMAHgIyM+sjNwHQMwR00AeCmZlBHgPAPQAzMyBHATDFQ0BmZn3kJgCm+TgAM7M+chQA/k1gM7NiuQmAxroaJGjv7KarOyrdHDOzihsyACStlrRN0hNFZUdKWitpY7qek8ol6UZJLZLWSzq76DErU/2NklaOz8sZ9HX07ArqH4Y3MxteD+BWYHm/squB+yJiCXBfug9wEbAkXVYBN0EWGMB1wLnAOcB1hdCYSB4GMjPrNWQARMT9wI5+xSuA29Lt24BLispvj8w6YLakY4ELgbURsSMidgJrGRgq466wJ5ADwMxs5HMAR0fEFoB0fVQqnw9sKqrXmsrKlQ8gaZWkZknNbW1tI2xeaT4YzMys11hPAqtEWQxSPrAw4uaIWBoRS5uamsa0cYVTQrsHYGY28gDYmoZ2SNfbUnkrsLCo3gJg8yDlE2pqffZyHQBmZiMPgDVAYU+elcDdReWXp72BlgG70xDRvcAFkuakyd8LUtmE6jkYzAFgZkbdUBUk3QG8BZgnqZVsb57PA3dJugJ4AXh3qn4PcDHQAuwHPggQETskfQ54MNX7bET0n1ged4VTQns3UDOzYQRARFxWZtH5JeoGcGWZ51kNrD6k1o2xqd4LyMysR26OBIbevYD2dXgvIDOzXAVA04xGALa+3F7hlpiZVV6uAmDBkVMBaN25v8ItMTOrvHwFwJxpALTufKXCLTEzq7ycBUChB+AAMDPLVQAcNXMK9bVi+952DnhXUDPLuVwFQG2NOG62ewFmZpCzAIDeYaBNngg2s5zLXwDM9kSwmRnkMQDmeFdQMzPIYwAc6TkAMzPIYwD4WAAzMyCHAXDsrCkAbN19oMItMTOrrNwFwBFT6wHYc+BghVtiZlZZuQuA6elHYfZ1dNHVXfJXKc3MciF3AVBbI6b7tNBmZvkLAIAZU7JewN4DDgAzy69cBsDMKdk8wN52B4CZ5VcuA2BGY9YD2OMegJnlWC4DYOaUQgB4TyAzy69cBkChB+AhIDPLs1EFgKSPS9og6QlJd0iaImmxpF9I2ijpnyQ1pLqN6X5LWr5oLF7ASPQEgIeAzCzHRhwAkuYD/wNYGhGnA7XApcBfAV+KiCXATuCK9JArgJ0R8SrgS6leRfTsBeQegJnl2GiHgOqAqZLqgGnAFuCtwHfS8tuAS9LtFek+afn5kjTK9Y9IYS8gTwKbWZ6NOAAi4kXgi8ALZB/8u4GHgF0RUfhkbQXmp9vzgU3psZ2p/tyRrn80ZnoOwMxsVENAc8i+1S8GjgOmAxeVqFo430Kpb/sDzsUgaZWkZknNbW1tI23eoHwgmJnZ6IaAfhd4NiLaIuIg8D3gPGB2GhICWABsTrdbgYUAafksYEf/J42ImyNiaUQsbWpqGkXzyus5DqDdu4GaWX6NJgBeAJZJmpbG8s8Hfgn8GPj9VGclcHe6vSbdJy3/fxFRkbOxzZjiA8HMzEYzB/ALssnch4HH03PdDHwa+ISkFrIx/lvSQ24B5qbyTwBXj6Ldo+I5ADOzbC+eEYuI64Dr+hU/A5xTou4B4N2jWd9Y6TkXkHsAZpZj+TwS2McBmJnlNAB8JLCZWc4DoKOTbv8qmJnlVC4DoLZGTGuoJcK/CmZm+ZXLAACfEdTMLLcBMNNHA5tZzuU2AGYUTgjnHoCZ5VRuA2Cm9wQys5zLbQDMmpr1AH6zr73CLTEzq4zcBsBJTdMB2Lh1b4VbYmZWGbkNgJOPmQnA01v3VLglZmaVkd8AODoLgKccAGaWU7kNgEVzp1NfKzbteIV93hPIzHIotwHQUFfDifNmANCyzfMAZpY/uQ0A6J0H8DCQmeVRvgPgqKwH8PRLDgAzy598B4B7AGaWY7kOgDMXzAbg4ed3crCru8KtMTObWLkOgGNmTeHEedPZ19HF+tbdlW6OmdmEynUAACw7aS4A6575TYVbYmY2sXIfAOelAPjZr7dXuCVmZhMr9wGw7MQsAJqf20l7Z1eFW2NmNnFGFQCSZkv6jqRfSXpS0hskHSlpraSN6XpOqitJN0pqkbRe0tlj8xJGZ96MRk45eibtnd088sKuSjfHzGzCjLYH8LfAv0bEq4EzgSeBq4H7ImIJcF+6D3ARsCRdVgE3jXLdY+YNaRjo57/2PICZ5ceIA0DSEcCbgVsAIqIjInYBK4DbUrXbgEvS7RXA7ZFZB8yWdOyIWz6GHABmlkej6QGcCLQB35D0iKR/kDQdODoitgCk66NS/fnApqLHt6ayilu2eC4SPLJpJ690eB7AzPJhNAFQB5wN3BQRrwP20TvcU4pKlMWAStIqSc2Smtva2kbRvOGbNa2e0447goNdQfPzOyZknWZmlTaaAGgFWiPiF+n+d8gCYWthaCddbyuqv7Do8QuAzf2fNCJujoilEbG0qalpFM07NOedNA+An3kYyMxyYsQBEBEvAZsknZKKzgd+CawBVqaylcDd6fYa4PK0N9AyYHdhqGgyOHfxkQA89NzOCrfEzGxi1I3y8R8FviWpAXgG+CBZqNwl6QrgBeDdqe49wMVAC7A/1Z00zlqYnRdo/Yu76Ozqpq4294dImFmVG1UARMSjwNISi84vUTeAK0ezvvE0d0YjJ8ydxvO/2c+vXtrD6fNnVbpJZmbjyl9zixR6AY9u8gFhZlb9HABFXucAMLMccQAUOev4OQA88oIngs2s+jkAipx67BE01NXw67Z9/OSpbUM/wMzsMOYAKNJQV8MHzlsEwKrbH+KnT0/MgWhmZpXgAOjnmotezQfOW0RHVzdXfethWrb594LNrDo5APqRxGfecSoXv/YY9rR3snL1gzzun4s0syrkACihpkZ88d1n8rrjZ/Pirlf4vZt+xjfXPU92KIOZWXVwAJQxraGOO1ct4/3LTqCjq5s/+8ETvP+WB/jh+s3sOXCw0s0zMxs1TeZvtUuXLo3m5uZKN4M1j23mmu+uZ186VXRdjfitE+Zw7olzOW7WFI46opGjZk7hqJmNTG2oZUp9LXU1Qip1AlQzs/El6aGIKHWWhr71HADDs2NfB3c/+iL3PL6Fh1/YRVf34NutRjClvpbGuhoa62qpqxX1tTXU14q6mnRdW0NdjWioy67r+iwv1OmtX19bk9UpqpuVifqamj7rqK2poUZQIyFlcxvF92ukdKFo+cA6vXWzOqUirVTQla5XoqxEzeHkZsnnmoB2DCgaw9dk1t/c6Q0j+iLpABhHu185yH9s3M6GzbvZtqedrS8foG1PO9v3tvNKRxftnd10DhEQZmZD+dXnljOlvvaQHzfcABjt2UBzadbUet5+xrG8/Yzyv2jZ2dXNgc5u2g+mQOgKOrq66ezObh/sykLiYGc3B7uDzq5uDnYFnd3dHCzc7rlfWN5bJ3uOwvMU1UnP2dUdBNAdQXdARNAdQUT/MnrKi++Xe0x/pb5AlIq+Ut8zokTN/vWG+/1kvNtR6vlKt214z2U2GTgAxkldbQ0zamuY0ehNbGaTk/cCMjPLKQeAmVlOOQDMzHLKAWBmllMOADOznHIAmJnllAPAzCynHABmZjk1qU8FIakNeH4UTzEP2D5GzalG3j6D8/YZnLfP4Cq5fU6IiKahKk3qABgtSc3DOR9GXnn7DM7bZ3DePoM7HLaPh4DMzHLKAWBmllPVHgA3V7oBk5y3z+C8fQbn7TO4Sb99qnoOwMzMyqv2HoCZmZXhADAzy6mqDABJyyU9JalF0tWVbs9kIOk5SY9LelRScyo7UtJaSRvT9ZxKt3MiSVotaZukJ4rKSm4TZW5M76n1ks6uXMsnRpntc72kF9P76FFJFxctuyZtn6ckXViZVk8MSQsl/VjSk5I2SPqfqfywev9UXQBIqgW+ClwEnApcJunUyrZq0vidiDiraN/kq4H7ImIJcF+6nye3Asv7lZXbJhcBS9JlFXDTBLWxkm5l4PYB+FJ6H50VEfcApP+xS4HT0mO+lv4Xq1Un8MmIeA2wDLgybYPD6v1TdQEAnAO0RMQzEdEB3AmsqHCbJqsVwG3p9m3AJRVsy4SLiPuBHf2Ky22TFcDtkVkHzJZU/kehq0CZ7VPOCuDOiGiPiGeBFrL/xaoUEVsi4uF0ew/wJDCfw+z9U40BMB/YVHS/NZXlXQA/kvSQpFWp7OiI2ALZGxo4qmKtmzzKbRO/r3pdlYYxVhcNG+Z2+0haBLwO+AWH2funGgNAJcq8ryu8MSLOJuuKXinpzZVu0GHG76vMTcBJwFnAFuBvUnkut4+kGcB3gY9FxMuDVS1RVvHtU40B0AosLLq/ANhcobZMGhGxOV1vA75P1j3fWuiGputtlWvhpFFum/h9BUTE1ojoiohu4Ov0DvPkbvtIqif78P9WRHwvFR9W759qDIAHgSWSFktqIJuYWlPhNlWUpOmSZhZuAxcAT5Btl5Wp2krg7sq0cFIpt03WAJenvTmWAbsLXf086Tdu/V/J3keQbZ9LJTVKWkw22fnARLdvokgScAvwZET8r6JFh9f7JyKq7gJcDDwN/Bq4ttLtqfQFOBF4LF02FLYJMJdsT4WN6frISrd1grfLHWTDGAfJvqFdUW6bkHXhv5reU48DSyvd/gptn2+m17+e7EPt2KL616bt8xRwUaXbP87b5k1kQzjrgUfT5eLD7f3jU0GYmeVUNQ4BmZnZMDgAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY59f8BF6Yi+JEx4I0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "graph(loss_history, 'Bias Only Played Loss History')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHfZJREFUeJzt3XucXGWd5/HPt6ov6XQ6CUmaEEhiIkQFVEAj43jF9QbIEH2Nl0Qd0VGZizizg6+Z1XFkGdTdVUeddRfdYVwHryDjiEYXFx3Xu4JE5RYiEiOXJuRKbt1J36p++8c51SkqdUvo7upT+b5fr3pVnVNPnXrq6epvP+c5p5+jiMDMzNpLrtUVMDOzyedwNzNrQw53M7M25HA3M2tDDnczszbkcDcza0MO9xlE0v+S9L5W16OSpCslfWEG1OM8SRsnu6w9PpLulfT8VtfDHsvhPo0k3S/pkKRBSXsk/R9Jy0rPR8SfRsT7p+B9z5C0XtI+SQckfU/Scyb7fSre8/np5xyUNCQpypYHJS0/2m1GxPcj4szJLnusJH1B0pikxVP5Pq0kaUDSeRXr3ibp+6XliHhyRPyowXZOk+R/qplGDvfp9wcRMQdYAmwH/sdUvpmkU4GfAHcBK4GTgRuBb0v6/al634j4UUTMST9rKWTnl9ZFxIMV9cxJysz3UVIf8CpgP/D6aX7vjul8v5ngePzMj1dmfpnaTUQMA18Bziitk3StpA+kj0+Q9E1JO9Ne/jclLS0r+2ZJW9Ke+O8kvaHGW10J/Cwi3hsRj0bEgYj4BPB54EPptlakPetLJD0oaZek91bbWLq38c6KdXdKeuXRtoGkH0t6v6SfAUPA8rRXuCn9XL+V9Lay8i+RdH/Z8oCkyyXdle6VXCep+2jLps+/R9I2SQ9LenvaHivqVP81wE7gvwCXVHyuDknvS+u/X9IGSSenzz1N0r9LejR9v79J139B0pUNPutfS7oLOJiu+7uy78BGSRdX1ONPJP06ff5uSWeln/PLFeU+Jekf6nzWusp795KeLemX6efeLukjabEfps+X9tyelf5Bv0LSA5J2pN//uWm509KfwVskPUjSGblZ0p9VvPc9ki461rq3tYjwbZpuwP3AS9LHs4HPAp8re/5a4APp44XAH6bl+oB/Bb6WPtdL0mN8crq8BDizxntuA95SZf2LgEK6/RVAAP8M9ABnASPA6WnZK4EvpI9fC9xatp2zgN1AV53PXdp+R8X6H6dtcjrQCXQAfwA8ERDwH4BDwNPT8i8B7i97/QBwC3BS2l6/Ad52DGUvAram9egFrkvru6LOZ/oBSbCfnLbjWWXPvQe4A1hF0oE6G1gAzCPZW/tLoBuYC5ybvuYLwJVl26hW/18AS4Gesp/FkvQ9Xg8MAovT59YBDwHPTNvyScCy9PWDwNy0XBewq7z+FZ9zADivYt3bgO9XKwPcBqxLH/cBv5c+Pg2Iiu1cmv4cVqZlvw78S3l54F9IvqM96Wf8SdnrnwnsoOJ75VvaPq2uwPF0S4NsENgLjKeB8rSy568lDfcqrz0b2JM+7k238YelX/Q67zkOnF9l/VPSX55TOBy+S8ue/zmwNn18JYfDvRt4FFiVLv8D8MkGdShtv1q4X9Hgtd8E3pE+rhZ4a8uWPwb8z2Mo+zng/VXaZkWNOq0EisBT0+XvAh8te/63wCuqvO6PgA01ttlMuL+pQVvdXXrftE7vqFHuO6R/8IFXAnfW2eZA2Xe2dDtI7XD/KXAFsLBiO9XC/QfApWXLZ5J0KnIcDvflZc/3pO//xHT5H4FPTNbvZ7vdPCwz/V4ZEfNJQvIy4AeSTqosJGm2pH9Kd1n3k+zWzpeUj4gh4HXAnwKPpEMlT6nxfrtIeneVlpAE1J6yddvKHh8E5lS+KCJGgBuAN6Zj5OtIhniO1UPlC5IuknRrOmyxF3gZsKjO6xvWuYmyJ1fU4zF1quJNwF0RcXe6/EXgDZLy6fIykoCvtAzY3GDb9VS21Zsl3SFpb9pWT+FwW9WqAyR7jG9MH7+Rxj+/iyJifukG/EWdsm8hGWq8V9LPJV1Yp+zJwANlyw+Q7En0l62b+MwRcYhkKLPU1mubqPtxy+HeIhFRiIivkuzSP69KkXcBTybZrZ0LvCBdr/T1N0fES0lC+tckQyrV/DvJ+HCl15KMxR88hup/FngD8GLgYET87Bi2UTJxBoWkHpJf3v9KMrwwH/g26WeeQo+QDFeULKtVUJJIwv1J6Zj5NuDDwGLg5Wmxh4BTq7y81npIjjnMLls+4g8+j22rJwKfAv6MpJc8n+R7UGqreu/1VeCZks4ELgC+VKPcUYuIeyNiLXAi8FHg3yTNKq97ma3AE8qWlwOjJMcySturfF3pu/cykj3Z2yar7u3G4d4iSqwBTgA2VSnSRzLevFfSAuA/l712saSLJfWS7MYOkvyRqObvgedI+qCkBZL6lBwQfRPwn46l7mmYF0l+eSez59RN0nPbCRTSA2UvnsTt13ID8FZJT5Y0G6j3vwbPIwn/1SRDZWcDT023UTqw+mngA5JOTX/OZ6c/w/UkB40vk9Qlaa6kc9PX3A68QsmB9CXU7x1DstcRJG0lJQeey/fePg38jaRz0jqsUnrabfoH/UaSYws/iYiHG7ZQkyT9kaRFEVEE9qV1LJKMjUf6R6nkOuByJQf0+4APAtelr63lxyTHZz6Ee+11Odyn3zckDZIcEP0gcElEVPtnm38kGWPcRXIg8P+WPZcj6dlvJRn/fiHw59XeLCLuIwmks0jG/B8hGat/eUT85HF8js8BTyMZK54UEbEX+CuS4HkUeDXJmPuUiohvkPSCfwjcR3LqKCR/OCtdAtwYERsjYlvpBvx3YI2k+cBHgK+RjHvvB64BZkXEPuClJO2/g+Rg4gvT7V5L8kf+AZKf9fUN6nwn8AmSYyOPkAT7rWXPX0cSgF9O6/BVko5EyWdJfn6THZAXApskHSA5HvO6iBiNiAMke2S3psNIq0n2Nr8M/AjYAhwgOdhcU9qT/zzJH9QvTnLd24qO3Osxa0zSm0gOhlUbUso0SU8Dfgl0N+hFZlbag74TOCkiBltdn6Mh6Y9JDi6f1+q6zGTuudtRS4cu/pykR9oWJL0qHSpZCPw34OttHOw54HLgSxkM9rb77k0Vh7sdFUkvJxnn3c4kHoibAd5BMgR2HzCcLrcdSfNIhmnOIzkekxmSXkHy3XuQZDjH6vCwjJlZG3LP3cysDbVsMp5FixbFihUrWvX2ZmaZ9Itf/GJXRPQ3KteycF+xYgUbNmxo1dubmWWSpAcal/KwjJlZW3K4m5m1IYe7mVkbcribmbUhh7uZWRtyuJuZtSGHu5lZGzrurygeETy89xAduRyL5nTRkU/+3j306EF+s/0AC+d0s23fIR4dGqO7I8cDu4cYKwYrF/aysr+XJy3uY15PJxHBzRu3seH+PZzQ28XiubM4sa+bhXO6GB4rcPtD+7hzYC99szpYNKebns48Ow6M0JET82Z3Mr+nixNmd048nj+7k5HxIlv3HuLhvYcoFIOezjyzOvP0dOXpzImRQpGRsSISE9sbLxQZLRT56ebd5PNi5cJe8jmxoLeLk+f3sHhuN3sPjnFgeHzK2rS3O8+p/XMoRnBgeJz9w2MMDo8TJPXs6crT05mnuyOHKi7DEQFDowV2HRhh99AIETC7u4Perjyzuzro7T58P14Itu8fJoB8TnTkRE6iIy868znmdHcwOl5krFBkTncH48Xg0FiBQ6MFhscKjBcfO/XG8FiBbfuGyeXE3Fmd9M3qoFAMRsaLjI7Xn0NsQW/yMzs0VmB4tMChsQKj40Vmd3dwaHSc3UOj9HZ1MFYoEsCp/XPo7shN1OfQWIHxgqcCOV4smT+LZyw/oXHBx+G4CfddgyNs2zfMKfN7ePDRg3z1lwOcOHcWP75vFz/bshtIgucpS/oYGhnnN9ubmyxPgpULexkZL/Lw3kNT+RHMrE1c9PQlPOP1DvfH5f5dQ/ztjXfx09/urlmmtytPT1cHuwZH+NWDewGY3ZXn7GXz2XtwjP6+bhbP7ebQWJHlC3rozOe4f9cQW3YN8etHDrBl1xAA/X3drHvWMobHi+zYP8yOAyPsHhxlVleeJyyYze+fupDR8SK7BkcYGimweG43xYC9h0bZd3CMPQdH2XtwjH2Hksed+Rwnz+/hlPk9VXt5XR05ujtyFCM4OFqgv6+brnyOQjF41soFdOZzDOw5SLEY7BoaZeveQ2zfP8L8nk7mz+48otc8WR4dGmXLziG6OnL0pT3gvu4OJDE8Vpj4HCPj1S8e1dOVZ9Gcbhb2dtORE0Oj4wyNjDM0WuDg6DgHRwoMjY6Tl1g8bxZ5ifFiUCgG48WgWAxGxgsMjhToyovOjhxDI+N05nPJHkNnnp7O3MReWklXPsfiubMIkj2OweFxOvKiK5+jM58jV2MQs1hMOg/7h8ces3fVlc9xcLTArM4ci+Z0c3C0QEdOFCLYvGOQCCb2Ynq68nTmp/pqgjZTnL1s/pS/R1uH+7fueoTLb7iDQ2MFZnflWXpCDw/sTi4Z+vrfW854IZjX08nbnr+S+bO72HlghN/uHKS3q4NTT+xldlfj5hkeK/DA7oMEwYqFvczqzDd8jZnZVGvbcP/SrQ/ytzfeBSS7QFeteSoLersYHS8SBN0dR4Zwf183/X3dR/U+szrzPPmkvkmps5nZZGnLcP/1tv1c+Y3ksqTvueApXPqCJ6J0DKKrwycImVn7a7twLxaDv/ryHYyOF1n7rGX8yQtPbXWVzMymXdt1Y2/ZsptNj+xnybxZvO+iM1pdHTOzlmi7cL/xVw8D8JpnLqW3u+12TMzMmtJW4X5otMC37t4GwJpzTmlxbczMWqetwv2bd25lcGScs5bO49T+Oa2ujplZy7RNuG/eMcjff+MeANadu7zFtTEza622CPeI4J3X/YrBkXFe8bQlvO5Zy1pdJTOzlmqLcP/N9kE2PbKfhb1dfOQ1T584p93M7HjVFuH+nXuSg6gvPv3EpqYMMDNrd20S7tsBeOkZJ7W4JmZmM0Pmw337/mHuGNjHrM4czzttUaurY2Y2I2Q+3H983y4AnnfaInq6PCOjmRm0Qbg/8Ggyhe/pS+a2uCZmZjNH5sN9YE8S7stOmN3impiZzRzZD/dHk0vbLT2hp8U1MTObObIf7mnPfal77mZmE5oKd0nnS7pX0mZJ767y/HJJ35P0K0l3Srpw8qt6pNHxItv2D5MTnDRv1nS8pZlZJjQMd0l54GrgAuAMYJ2kyonS/w64ISLOAdYCn5zsilazbd8wxYCT5s7yFZbMzMo0k4jnApsjYktEjALXA2sqygRQOl1lHrB18qpYm4dkzMyqaybcTwEeKlseSNeVuxJ4o6QB4CbgndU2JOlSSRskbdi5c+cxVPexBvb4YKqZWTXNhHu1WbiiYnkdcG1ELAUuBD4v6YhtR8Q1EbE6Ilb39/cffW0rHO65O9zNzMo1E+4DQPkcuks5ctjlrcANABHxM2AWMOVzATw00XP3sIyZWblmwv02YJWklZK6SA6Yrq8o8yDwYgBJp5OE++Mfd6lhz9AoL/3YD1h/R/I3xj13M7PHahjuETEOXAbcDGwiOStmo6SrJF2cFnsX8HZJdwDXAW+OiMqhm0lz18P7uG/HIIVicGJfN2eeMm+q3srMLJOamvw8Im4iOVBavu6Kssf3AM+d3KrVVkz/bjzn1IV87o/PpSPv0yDNzMplMhVL+wRdHTkHu5lZFZlMxlLPPefL6ZmZVZXRcE/uc852M7OqMhruSbr7QthmZtVlM9yLpWGZFlfEzGyGyma4TwzLON3NzKrJaLinPXd33c3Mqsp2uLvnbmZWVSbDPXy2jJlZXZkMd/fczczqy2S4F4qlUyFbXBEzsxkqk+EePlvGzKyuTIZ7aVgm73A3M6sqo+Ge3OcyWXszs6mXyXj09ANmZvVlMtwjPP2AmVk9mQz3QtGnQpqZ1ZPJcPfcMmZm9WU03N1zNzOrJ5Ph7ukHzMzqy2S4e1ZIM7P6MhnuhfD0A2Zm9WQy3D39gJlZfZkM99Jl9jz9gJlZddkMdx9QNTOrK6Ph7ukHzMzqyWS4h89zNzOrK5PhXvDcMmZmdWUy3A9P+et0NzOrJqPh7mEZM7N6Mhnunn7AzKy+TIZ70VP+mpnVlc1wT3vuznYzs+oyGu7uuZuZ1ZPxcG9xRczMZqimwl3S+ZLulbRZ0rtrlHmtpHskbZT0pcmt5mOVwj3vdDczq6qjUQFJeeBq4KXAAHCbpPURcU9ZmVXAe4DnRsQeSSdOVYWhfMzd4W5mVk0zPfdzgc0RsSUiRoHrgTUVZd4OXB0RewAiYsfkVvOxPP2AmVl9zYT7KcBDZcsD6bpyTwKeJOknkm6RdH61DUm6VNIGSRt27tx5bDUGisXk3qMyZmbVNRPu1SI0KpY7gFXAecA64NOS5h/xoohrImJ1RKzu7+8/2rpOKLjnbmZWVzPhPgAsK1teCmytUubrETEWEb8D7iUJ+ylR9GX2zMzqaibcbwNWSVopqQtYC6yvKPM14EUAkhaRDNNsmcyKlitNP+CzZczMqmsY7hExDlwG3AxsAm6IiI2SrpJ0cVrsZmC3pHuA7wF/HRG7p6rS/icmM7P6Gp4KCRARNwE3Vay7ouxxAJentynn6QfMzOrL+H+oOt3NzKrJZrh7Vkgzs7qyGe6eW8bMrK6Mhnty78vsmZlVl8lw9/QDZmb1ZTLci77MnplZXRkNd/fczczqyWS4F4qefsDMrJ5MhrunHzAzqy+T4e5hGTOz+jId7s52M7PqMhruyb177mZm1WUz3D39gJlZXdkMd08/YGZWV0bDPbn39ANmZtVlMtw9/YCZWX2ZDHdPP2BmVl9Gw909dzOzejIZ7p5+wMysvkyGe/g8dzOzujIZ7qVhGc8tY2ZWXabD3dluZlZdJsO9NCwjD8uYmVWVyXD32TJmZvVlMtwLHpYxM6srk+FeLCb37rmbmVWXyXCfmH7AXXczs6oyGe6efsDMrL6MhrsPqJqZ1ZPRcE/une1mZtVlNNzdczczq8fhbmbWhrIZ7um4TN7hbmZWVSbDfWL6gUzW3sxs6jUVj5LOl3SvpM2S3l2n3KslhaTVk1fFI3lYxsysvobhLikPXA1cAJwBrJN0RpVyfcBfALdOdiUr+Tx3M7P6mum5nwtsjogtETEKXA+sqVLu/cCHgeFJrF9VBffczczqaibcTwEeKlseSNdNkHQOsCwivllvQ5IulbRB0oadO3cedWVLwuFuZlZXM+FeLUFj4kkpB3wceFejDUXENRGxOiJW9/f3N1/LCh6WMTOrr5lwHwCWlS0vBbaWLfcBTwW+L+l+4NnA+qk8qOoDqmZm9TUT7rcBqyStlNQFrAXWl56MiH0RsSgiVkTECuAW4OKI2DAVFY6IsisxTcU7mJllX8Nwj4hx4DLgZmATcENEbJR0laSLp7qCR9YnuZd8mT0zs1o6mikUETcBN1Wsu6JG2fMef7Vq85kyZmaNZe5/PEvj7Z56wMystsyFu8fbzcway1y4+0wZM7PGMhjuyb3PcTczqy1z4V4ouuduZtZI5sK9NPWAs93MrLbMhXtpWCbvcRkzs5oyGO4eljEzaySz4e7/TjUzqy1z4R4+W8bMrKHMhbvPljEzayxz4X54zL3FFTEzm8EyF+4TwzJOdzOzmjIX7j5bxsyssQyGe3LvjruZWW0ZDHf33M3MGsleuBc9/YCZWSPZC/eJYRmnu5lZLRkM9/RKTB50NzOrKbPh7ukHzMxqy1y4e/oBM7PGMhfuPlvGzKyxzIX74bllWlwRM7MZLHPhXjpbxmPuZma1ZS7cw2fLmJk1lLlw9/QDZmaNZTDcfSqkmVkjmQ1399zNzGrLXrgXk3ufCmlmVlv2wt0HVM3MGspsuHvM3cystsyFu6cfMDNrLHPh7ukHzMway2C4J/fuuZuZ1dZUuEs6X9K9kjZLeneV5y+XdI+kOyV9V9ITJr+qiULRY+5mZo00DHdJeeBq4ALgDGCdpDMqiv0KWB0RTwe+Anx4sitaMjH9gMPdzKymZnru5wKbI2JLRIwC1wNrygtExPci4mC6eAuwdHKredjEsEzmBpTMzKZPMxF5CvBQ2fJAuq6WtwLfejyVqsenQpqZNdbRRJlqKRpVC0pvBFYDL6zx/KXApQDLly9vsoqP5bNlzMwaa6bnPgAsK1teCmytLCTpJcB7gYsjYqTahiLimohYHRGr+/v7j6W+nlvGzKwJzYT7bcAqSSsldQFrgfXlBSSdA/wTSbDvmPxqHua5ZczMGmsY7hExDlwG3AxsAm6IiI2SrpJ0cVrsI8Ac4F8l3S5pfY3NPW4eljEza6yZMXci4ibgpop1V5Q9fskk16tOXZJ7D8uYmdWWuRMK3XM3M2ssg+Ge3Ps8dzOz2jIXkQWf525m1lDmwj18KqSZWUOZC/di0XPLmJk1kr1wT8fcPSxjZlZbBsPdZ8uYmTWSuXD3ee5mZo1lLtxLZ8vknO5mZjVlLtwPT/nb4oqYmc1gmQv30rCMz5YxM6stc+FeOhXSB1TNzGrLXrj7gKqZWUMZDHdPP2Bm1khmw93DMmZmtWU23POZq7mZ2fTJXER6+gEzs8YyGO4eljEzayRz4e7pB8zMGstcuPs8dzOzxjIX7gVPP2Bm1lDmwn1i+gGPy5iZ1ZS5cPcBVTOzxjIc7i2uiJnZDJbBcE/ufZ67mVltmQv38LCMmVlDmQv3QtHDMmZmjWQu3Cem/HW6m5nVlMFw97CMmVkjmQt3Tz9gZtZY5sLdPXczs8YyF+6lA6rOdjOz2jIX7oeHZZzuZma1ZC7cD1+JyeFuZlZLZsPd2W5mVltT4S7pfEn3Stos6d1Vnu+W9OX0+VslrZjsipZ4+gEzs8YahrukPHA1cAFwBrBO0hkVxd4K7ImI04CPAx+a7IqWePoBM7PGmum5nwtsjogtETEKXA+sqSizBvhs+vgrwIs1RV1rTz9gZtZYM+F+CvBQ2fJAuq5qmYgYB/YBCys3JOlSSRskbdi5c+cxVbhvVicLervo6sjc4QIzs2nT0USZan3kOIYyRMQ1wDUAq1evPuL5Znxi3TnH8jIzs+NKM93fAWBZ2fJSYGutMpI6gHnAo5NRQTMzO3rNhPttwCpJKyV1AWuB9RVl1gOXpI9fDfy/KB35NDOzaddwWCYixiVdBtwM5IHPRMRGSVcBGyJiPfC/gc9L2kzSY187lZU2M7P6mhlzJyJuAm6qWHdF2eNh4DWTWzUzMztWPuXEzKwNOdzNzNqQw93MrA053M3M2pBadcaipJ3AA8f48kXArkmsTrtx+zTmNqrP7VNfK9vnCRHR36hQy8L98ZC0ISJWt7oeM5XbpzG3UX1un/qy0D4eljEza0MOdzOzNpTVcL+m1RWY4dw+jbmN6nP71Dfj2yeTY+5mZlZfVnvuZmZWh8PdzKwNZS7cG12s+3gk6X5Jd0m6XdKGdN0CSd+RdF96f0Kr6zldJH1G0g5Jd5etq9oeSnwi/T7dKekZrav59KjRPldKejj9Dt0u6cKy596Tts+9kl7emlpPH0nLJH1P0iZJGyX9Zbo+U9+hTIV7kxfrPl69KCLOLjv39t3AdyNiFfDddPl4cS1wfsW6Wu1xAbAqvV0KfGqa6thK13Jk+wB8PP0OnZ3OBEv6+7UWODN9zSfT38N2Ng68KyJOB54NvCNth0x9hzIV7jR3sW5LlF+0/LPAK1tYl2kVET/kyCuB1WqPNcDnInELMF/SkumpaWvUaJ9a1gDXR8RIRPwO2Ezye9i2IuKRiPhl+vgAsInkOtGZ+g5lLdybuVj38SiAb0v6haRL03WLI+IRSL6swIktq93MUKs9/J067LJ0WOEzZcN4x3X7SFoBnAPcSsa+Q1kL96YuxH0cem5EPINk9/Adkl7Q6gpliL9TiU8BpwJnA48AH03XH7ftI2kO8G/Af4yI/fWKVlnX8jbKWrg3c7Hu405EbE3vdwA3kuw2by/tGqb3O1pXwxmhVnv4OwVExPaIKEREEfhnDg+9HJftI6mTJNi/GBFfTVdn6juUtXBv5mLdxxVJvZL6So+BlwF389iLll8CfL01NZwxarXHeuBN6RkPzwb2lXa9jycVY8SvIvkOQdI+ayV1S1pJctDw59Ndv+kkSSTXhd4UER8reypb36GIyNQNuBD4DfBb4L2trk+rb8ATgTvS28ZSmwALSY7o35feL2h1XaexTa4jGVoYI+lVvbVWe5DsUl+dfp/uAla3uv4tap/Pp5//TpKwWlJW/r1p+9wLXNDq+k9D+zyPZFjlTuD29HZh1r5Dnn7AzKwNZW1YxszMmuBwNzNrQw53M7M25HA3M2tDDnczszbkcDcza0MOdzOzNvT/AXnAXSQfX8weAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "graph(train_accuracy_history, 'Bias Only Training Accuracy History')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions, labels = generate_outputs(played_test_data, binary_played_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5766967438211064"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy(predictions, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('./data/bias_only_played_itemBiases', itemBiases)\n",
    "np.save('./data/bias_only_played_loss', loss_history)\n",
    "np.save('./data/bias_only_played_train_accuracy', train_accuracy_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Complete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_history = []\n",
    "train_accuracy_history = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "complete_played_res, complete_played_f, complete_played_d = sy.optimize.fmin_l_bfgs_b(cost, \n",
    "                                [0.0]*nItems + # Initialize beta\n",
    "                                [random.random() * 0.1 - 0.05 for k in range(K*(nUsers + nItems))], # Gamma\n",
    "                                derivative, args=[0.001, played_train_data, played_train_labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'grad': array([ 0.00000000e+00,  1.76730285e-07,  0.00000000e+00, ...,\n",
       "        -4.37533362e-09, -6.22876887e-09,  8.05804503e-09]),\n",
       " 'task': b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL',\n",
       " 'funcalls': 307,\n",
       " 'nit': 278,\n",
       " 'warnflag': 0}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "complete_played_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "unpack(complete_played_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEICAYAAAC55kg0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuYXHWd5/H3p69JCBAgDQYSCJegXNSIWWDX0WUF5TIq6KNOMiqMuhN1YHd40HFgdJRBmXVUvDCrMCBZwBGQERB0YAVdFB252GAMCREJ19xIWkJIINfu/u4f51ed09VV1ffudJ3P63nqqarf+Z0639MnqW/9LuccRQRmZlZMDeMdgJmZjR8nATOzAnMSMDMrMCcBM7MCcxIwMyswJwEzswJzErBRJ+kZSafsBnGEpCPGeJt/IelXY7nNkSLpSkl/P95x2OhyEqhjkv5cUruklyWtlXSXpD8Z77iqkTQ7fVE3DXP9l9PjGUkXjnScI2W4+zvMbV8r6Yu14omIj0fEFwbwWbtFkrehcRKoU5IuAL4B/CNwAHAw8G3gzPGMa4xMi4ipwALgc5JOG++ArLLxSIDWm5NAHZK0N3AJcG5E3BoRr0TEzoj4UUT8TarTKukbktakxzcktaZlJ0laJenTktanVsRZks6Q9AdJGyT9XW57F0v6gaTvS9os6RFJr68SW4OkCyU9KekFSTdL2jctvi89b0y/5P9zWucjkpZLelHSTyQdMpC/Q0TcDywDjq0Qx59K+q2kTZJWSro4t+zfJf2PsvpLJJ2VXr9G0j3p7/C4pPfn6u0n6Y70uQ8Bhw8k1grx1To+0yX9WNLGFMMvJTWkZX8raXU6Do9LOnko20+f1dNaqLZNSd8l+4Hxo3TMPp3qv0vSslT/55KOyn3uMynOJcArkv5G0i1l2/5nSd8Yauw2CBHhR509gNOATqCpRp1LgAeA/YE24NfAF9Kyk9L6nwOagb8EOoAbgD2BY4BtwGGp/sXATuC9qf6ngKeB5rT8GeCU9Pr8tN2ZQCvwL8CNadlsIPJxA2cBK4CjgCbgs8Cvq+xTz/qAgDcBW4CT0/IAjsjt42vJfgi9DlgHnJWWvR94MPe5rwdeAFqAPYCVwIfTdo4D/ggck+reBNyc6h0LrAZ+1V+8gzw+/wu4Mv2tm4E3p/19dYrtwNznH15l29cCX6wVT75OtW2WH9/0/kjgFeBtqe6n0zFsydVfDMwCJgMzUv1paXkTsB5443j/XyrCY9wD8GMUDip8AHi+nzpPAmfk3p8KPJNenwRsBRrT+z3Tl8MJufoP5740LwYeyC1rANYCb07ve74kgOWkL+X0fgZZAmmq9KUI3AV8tOyztwCHVNin0vobgRfTtv5nbnlPEqiw7jeAr6fXrcAGYE56/1Xg2+n1nwG/LFv3X4DPA41pX16TW/aPDC0J1Do+lwC3l+8LcET68jyFlIBrHP9ryRL5xtxjE9WTQMVtlh/f9P7vgZvLjtlq4KRc/Y+UfcZdwF+m1+8AHhvv/0dFebg7qD69AEzvp7/1QODZ3PtnU1nPZ0REV3q9NT2vyy3fCkzNvV9ZehER3cCqss8rOQS4LXUTbCT7ou4iG7eo5BDgm7n6G8h+9R5UY9+mR8Q+EXFURFxeqYKkEyTdK6lD0kvAx4HpKf7tZL/mP5i6WRYA383Fc0IpnhTTB4BXkf1ib8r/Lej9Nx6MWsfnK2S/rO+W9FRp8DsiVpC1tC4G1ku6SVKlY1Dy1YiYVnqQtYiqqbjNgcSe/j2spPcxW1m2znXAB9PrD7Lr722jzEmgPt1P9ivvrBp11pB9oZUcnMqGalbpRfrinFnl81YCp+e/fCJiUkSsJvsVWqn+x8rqT46IXw8jVsi6tu4AZkXE3mRdHcotv47sy/1kYEtk4wuleH5RFs/UiPgEWZdZJ7m/BdnfdSiqHp+I2BwRn4yIw4B3AheU+v4j4oaI+JO0bgD/NMTt91Jrm/Q9br1ilySyv8nq/EeWrfND4HWSjiVrCXxvJOK2/jkJ1KGIeImsP/9baUB3iqRmSadL+nKqdiPwWUltkqan+v86jM2+UdJ7UuvjfGA7WZ92uSuBS0uDu2n7pRlLHUA3cFhZ/YskHZPq7y3pfcOIs2RPYENEbJN0PPDn+YXpS78buIzev0p/DBwp6UPpb9os6T9JOiq1nG4FLk5/86OBcwYQS6ukSblHAzWOj6R3SDoifbluImtJdUl6taS3pgHkbWStta7KmxycattMi9fR+5jdDPyppJMlNQOfJPv3UDVxR8Q24AdkyfmhiHhuJOK2/jkJ1KmI+BpwAdlAagfZL9jzyH5xAXwRaAeWAI8Cj6SyobqdrL/8ReBDwHsiYmeFet8k+wV+t6TNZInihBTzFuBS4D9SV8uJEXEb2a/ZmyRtApYCpw8jzpK/Ai5JMXyO7Iur3PVkg8c9yTEiNgNvB+aT/eJ9PsXXmqqcR9ZN9jxZn/r/GUAsL5N9YZceb6X28ZkD/DStdz/ZeMXPUwxfIhuofp5sULlnFtcwVdsmZIPGn03H7FMR8ThZl84/p1jeCbwzInb0s43ryP7e7goaQ6XRfbMhS9Mrj4iID/ZXdyKRdDawMHWv2CiTdDDwe+BVEbFpvOMpCrcEzCqQNIWstXDVeMdSBKkL7ALgJieAseUkYFZG0qlkXWjryPqobRRJ2oNsnOFtZFNtbQy5O8jMrMDcEjAzK7Dd/uJN06dPj9mzZ493GGZmE8bDDz/8x4hoG0jd3T4JzJ49m/b29vEOw8xswpA04DPV3R1kZlZgTgJmZgXmJGBmVmBOAmZmBeYkYGZWYE4CZmYF5iRgZlZg/SYBSYuU3Wx8aa7s+5IWp8czkhan8tmStuaWXZlb542SHpW0QtLl6brko2LZmpf40DUPctGtS0ZrE2ZmdWEgJ4tdC/xvsmurAxARf1Z6Leky4KVc/ScjYm6Fz7kCWEh2/fg7yW6GftfgQ+6fEL984o8c3rbHaHy8mVnd6LclEBH3kd3XtY/0a/79ZHdBqkrSDGCviLg/sivWXU/tWx8Oy8H7TQFg5Ytb6e72BfLMzKoZ7pjAm4F1EfFEruxQSb+V9AtJb05lB5HdeLxkFTVuFC5poaR2Se0dHR2DDmpqaxPTp7awo7Ob5zdtG/T6ZmZFMdwksIDerYC1wMER8QayG0TcIGkvet/Au6TqT/SIuCoi5kXEvLa2AV0DqY+D981aA8++sGVI65uZFcGQk0C6ofh7gO+XyiJie0S8kF4/DDwJHEn2y39mbvWZZPdnHTWH7JeNBzy34ZXR3IyZ2YQ2nJbAKcDvI6Knm0dSm6TG9PowsptTPxURa4HNkk5M4whnk92YfNS4JWBm1r+BTBG9EbgfeLWkVZI+mhbNp++A8FuAJZJ+B/wA+HhElAaVPwF8B1hB1kIYlZlBJYekweFnNzgJmJlV0+8U0YhYUKX8LyqU3QLcUqV+O3DsIOMbslmpJbD6xa1jtUkzswmnbs8Y3mtSMwCvbO8c50jMzHZfdZsEprQ0ArBlR9c4R2Jmtvuq2yQwuScJuCVgZlZN3SaBPVqy4Q63BMzMqqvbJDCpuQEJtnd20+VLR5iZVVS3SUASk5vdJWRmVkvdJgGAKalLaKu7hMzMKqrzJOAZQmZmtRQiCbzi7iAzs4oKkQTcHWRmVlmdJwFPEzUzq6Wuk4BPGDMzq62uk8AeHhg2M6uprpPAZHcHmZnVVNdJYIq7g8zMaqrrJODuIDOz2uo6Cbg7yMystrpOAu4OMjOrrSBJwC0BM7NKBnKj+UWS1ktamiu7WNJqSYvT44zcsoskrZD0uKRTc+WnpbIVki4c+V3pq+dkse1OAmZmlQykJXAtcFqF8q9HxNz0uBNA0tHAfOCYtM63JTVKagS+BZwOHA0sSHVH1ZTW1BLY6SRgZlZJU38VIuI+SbMH+HlnAjdFxHbgaUkrgOPTshUR8RSApJtS3ccGHfEgTCndT8A3mzczq2g4YwLnSVqSuov2SWUHAStzdValsmrlFUlaKKldUntHR8eQA5yUksD2zu4hf4aZWT0bahK4AjgcmAusBS5L5apQN2qUVxQRV0XEvIiY19bWNsQQoaUp270dTgJmZhX12x1USUSsK72WdDXw4/R2FTArV3UmsCa9rlY+apobUxLochIwM6tkSC0BSTNyb98NlGYO3QHMl9Qq6VBgDvAQ8BtgjqRDJbWQDR7fMfSwB6bVLQEzs5r6bQlIuhE4CZguaRXweeAkSXPJunSeAT4GEBHLJN1MNuDbCZwbEV3pc84DfgI0AosiYtmI702ZUneQxwTMzCobyOygBRWKr6lR/1Lg0grldwJ3Diq6YWpJ3UE73R1kZlZRXZ8x7IFhM7PaipEE3BIwM6uorpNAU4OQoKs76OquOiPVzKyw6joJSNo1TdRdQmZmfdR1EgBodRIwM6uq7pNAzzTRLl9EzsysXGGSwM4ujwmYmZUrTBJwd5CZWV/1nwQ8JmBmVlX9JwG3BMzMqipOEvDAsJlZH3WfBErnCfgicmZmfdV9EvDlpM3Mqqv7JLDrSqKeImpmVq7+k4BbAmZmVRUnCXhg2Mysj/pPAj5PwMysqvpPAu4OMjOrqu6TgKeImplV128SkLRI0npJS3NlX5H0e0lLJN0maVoqny1pq6TF6XFlbp03SnpU0gpJl0vS6OxSb62+u5iZWVUDaQlcC5xWVnYPcGxEvA74A3BRbtmTETE3PT6eK78CWAjMSY/yzxwVPVcR7fQUUTOzcv0mgYi4D9hQVnZ3RHSmtw8AM2t9hqQZwF4RcX9EBHA9cNbQQh6cnoFhzw4yM+tjJMYEPgLclXt/qKTfSvqFpDensoOAVbk6q1JZRZIWSmqX1N7R0TGs4DwwbGZW3bCSgKTPAJ3A91LRWuDgiHgDcAFwg6S9gEr9/1X7ZyLiqoiYFxHz2trahhOik4CZWQ1NQ11R0jnAO4CTUxcPEbEd2J5ePyzpSeBIsl/++S6jmcCaoW57MFo8MGxmVtWQWgKSTgP+FnhXRGzJlbdJakyvDyMbAH4qItYCmyWdmGYFnQ3cPuzoB6DFU0TNzKrqtyUg6UbgJGC6pFXA58lmA7UC96SZng+kmUBvAS6R1Al0AR+PiNKg8ifIZhpNJhtDyI8jjBp3B5mZVddvEoiIBRWKr6lS9xbglirL2oFjBxXdCPBlI8zMqqv7M4Z7zhPwmICZWR+FSQIeGDYz66v+k4C7g8zMqqr7JNDa3Ah4dpCZWSV1nwQmNWe7uG2nLxthZlau/pNAU9YS2LbTLQEzs3L1nwSaS0nALQEzs3IFSALZLm51EjAz66MASSANDLs7yMysj7pPAvk7i3V1+8YyZmZ5dZ8EJPV0CW3vdJeQmVle3ScByA8Ou0vIzCyvGEmgyTOEzMwqKUYS8AwhM7OKCpIE3BIwM6ukYEnAYwJmZnkFSQJpdpBbAmZmvRQkCaSWgKeImpn1Uowk4IvImZlVNKAkIGmRpPWSlubK9pV0j6Qn0vM+qVySLpe0QtISScfl1jkn1X9C0jkjvzuV9cwO2uGWgJlZ3kBbAtcCp5WVXQj8LCLmAD9L7wFOB+akx0LgCsiSBvB54ATgeODzpcQx2twdZGZW2YCSQETcB2woKz4TuC69vg44K1d+fWQeAKZJmgGcCtwTERsi4kXgHvomllHh2UFmZpUNZ0zggIhYC5Ce90/lBwErc/VWpbJq5aPO5wmYmVU2GgPDqlAWNcr7foC0UFK7pPaOjo5hB+QpomZmlQ0nCaxL3Tyk5/WpfBUwK1dvJrCmRnkfEXFVRMyLiHltbW3DCDGza0zA3UFmZnnDSQJ3AKUZPucAt+fKz06zhE4EXkrdRT8B3i5pnzQg/PZUNuomNXl2kJlZJU0DqSTpRuAkYLqkVWSzfL4E3Czpo8BzwPtS9TuBM4AVwBbgwwARsUHSF4DfpHqXRET5YPOo8JiAmVllA0oCEbGgyqKTK9QN4Nwqn7MIWDTg6EbI5BZ3B5mZVVKIM4ZbfT8BM7OKCpEESrODnATMzHorSBLIWgLbfbKYmVkvhUoCvrOYmVlvBUkC7g4yM6ukEElgsi8gZ2ZWUSGSgC8gZ2ZWWTGSgKeImplVVIgk0NpzATm3BMzM8oqRBJoakGBHVzdd3RUvXGpmVkiFSAKSaG3yDCEzs3KFSALgi8iZmVVSmCQw2fcUMDProzBJwC0BM7O+CpMEPCZgZtZXYZKAWwJmZn0VKAmUWgIeEzAzKylQEnBLwMysXGGSwGRfP8jMrI/CJAG3BMzM+hpyEpD0akmLc49Nks6XdLGk1bnyM3LrXCRphaTHJZ06MrswMD1jAr6ctJlZj6ahrhgRjwNzASQ1AquB24APA1+PiK/m60s6GpgPHAMcCPxU0pERMSbfyrtuNu/uIDOzkpHqDjoZeDIinq1R50zgpojYHhFPAyuA40do+/1yd5CZWV8jlQTmAzfm3p8naYmkRZL2SWUHAStzdValsj4kLZTULqm9o6NjRAL0LSbNzPoadhKQ1AK8C/i3VHQFcDhZV9Fa4LJS1QqrV7yuc0RcFRHzImJeW1vbcEME8rODnATMzEpGoiVwOvBIRKwDiIh1EdEVEd3A1ezq8lkFzMqtNxNYMwLbHxDfYtLMrK+RSAILyHUFSZqRW/ZuYGl6fQcwX1KrpEOBOcBDI7D9AXF3kJlZX0OeHQQgaQrwNuBjueIvS5pL1tXzTGlZRCyTdDPwGNAJnDtWM4Mg1xLwpaTNzHoMKwlExBZgv7KyD9Wofylw6XC2OVSlKaJbd7glYGZWUqAzhtPN5n2ymJlZjwIlAc8OMjMrV5gk0JJuKrOjq+KsVDOzQipMEmhuyHa1s8sDw2ZmJcVJAk3ZuWo7nQTMzHoUJgk09bQE3B1kZlZSmCTQ0lgaE3BLwMyspDBJoKkx6w5yS8DMbJfCJIHm1BLo7HZLwMyspEBJIGsJ7PBlI8zMehQoCZRaAu4OMjMrKUwSKI0JeIqomdkuhUkCpZPFdnYFEW4NmJlBgZJAQ4NobEgzhNwlZGYGFCgJADQ1eJqomVleoZKATxgzM+utUElg1wljTgJmZlCwJFCaJrrT3UFmZkBhk4BbAmZmMAJJQNIzkh6VtFhSeyrbV9I9kp5Iz/ukckm6XNIKSUskHTfc7Q9Gs88VMDPrZaRaAv8tIuZGxLz0/kLgZxExB/hZeg9wOjAnPRYCV4zQ9gekyWcNm5n1MlrdQWcC16XX1wFn5cqvj8wDwDRJM0Yphj7cHWRm1ttIJIEA7pb0sKSFqeyAiFgLkJ73T+UHAStz665KZb1IWiipXVJ7R0fHCISY2dUd5JaAmRlA0wh8xpsiYo2k/YF7JP2+Rl1VKOvzjRwRVwFXAcybN2/EvrF7LiLnloCZGTACLYGIWJOe1wO3AccD60rdPOl5faq+CpiVW30msGa4MQxU6YxhnyxmZpYZVhKQtIekPUuvgbcDS4E7gHNStXOA29PrO4Cz0yyhE4GXSt1GY2FXS8DdQWZmMPzuoAOA2ySVPuuGiPi/kn4D3Czpo8BzwPtS/TuBM4AVwBbgw8Pc/qB4iqiZWW/DSgIR8RTw+grlLwAnVygP4NzhbHM4mnzGsJlZL4U6Y7jFU0TNzHopVBLouYCcbzZvZgYULAn0nCzW6e4gMzMoXBJIA8NuCZiZAYVLAqWWgJOAmRkULAk0NfgCcmZmeYVKAs1NvnaQmVlesZJAg6eImpnlFSsJ+AJyZma9FCoJlM4T2OHuIDMzoGBJoDRF1C0BM7NMwZKAxwTMzPIKlQR6LiDnKaJmZkDBkkBL6YxhnyxmZgYULAn4ZDEzs94KlQSam7Ld9e0lzcwyxUoCDZ4dZGaWV6wk4DuLmZn1Uqgk0OR7DJuZ9TLkJCBplqR7JS2XtEzSX6fyiyWtlrQ4Pc7IrXORpBWSHpd06kjswGD49pJmZr0N50bzncAnI+IRSXsCD0u6Jy37ekR8NV9Z0tHAfOAY4EDgp5KOjIiuYcQwKD0Dw54iamYGDKMlEBFrI+KR9HozsBw4qMYqZwI3RcT2iHgaWAEcP9TtD8Xk5kYAtu50EjAzgxEaE5A0G3gD8GAqOk/SEkmLJO2Tyg4CVuZWW0WVpCFpoaR2Se0dHR0jESIAk1tSEtjROWKfaWY2kQ07CUiaCtwCnB8Rm4ArgMOBucBa4LJS1QqrV5ymExFXRcS8iJjX1tY23BB7TCklgZ1j1gNlZrZbG1YSkNRMlgC+FxG3AkTEuojoiohu4Gp2dfmsAmblVp8JrBnO9gdrSnM2BLJlh5OAmRkMb3aQgGuA5RHxtVz5jFy1dwNL0+s7gPmSWiUdCswBHhrq9odiUku2u9vcEjAzA4Y3O+hNwIeARyUtTmV/ByyQNJesq+cZ4GMAEbFM0s3AY2Qzi84dy5lBkE0RbWwQO7uCnV3dPSePmZkV1ZCTQET8isr9/HfWWOdS4NKhbnO4JDGluZHN2zvZsqOLvSc7CZhZsRXuW3BSGhx2l5CZWQGTQGmGkAeHzcwKmARKJ4xt8bkCZmYFTALuDjIz61G4JODuIDOzXQqXBCbXOGFs284uPvvDR7n/yRfGOiwzs3FRvCRQozvou/c/y78+8BwLrn5grMMyMxsXhUsCU5qrdwet3rh1rMMxMxtXhUsCkwc4JvDfr2vn7mXPj0VIZmbjprBJoFJ3UMSui5r+dPk6Fn734TGLy8xsPBQuCUypcZ7ANt9sxswKpnBJoFZ30Itbdox1OGZm46qwSaBSd1ClJOCTysysnhUuCdQ6WWzDK32TwPpN20c9JjOz8VK4JFDrZLGNW3b2KVu/eduox2RmNl6Gc1OZCanUHfTK9l0Dw1t2dPLFf1/OC5VaApvdEjCz+lW4lsAh+05BggeeeoGr73uKTdt2cvV9T3PDg89VrN/hJGBmdaxwSWD29D0496Qj6A649M7lnPSVn3PDQ89Wrb9+8zZWbtjCB77zALc+smoMIzUzG32F6w4COP+UOUyf2sIPF69h8cqNNet+694n+da9TwLwHyteoLMrmDFtEvtMaWGfPVrYd0pLTxeTmdlEo/xZsmOyQek04JtAI/CdiPhSrfrz5s2L9vb2UYklIvjRkrVc86unOWDPVu5+bB377dFScWyglulTWzhs+lQOnDaJNRu38ZoZe3L0jL1oaWpAAiEkaG5soKWxgaZG0dSQ3fS+qVHZc0PpubQObNy6k8XPbeS1M/embc9WYNdNnSXlXsPytZv5ybLnedfcAzl8+tRe8anSnaAHsFwVFlT7qEqfoSq1K9et9sHD+1yziWza5GaaGgffYSPp4YiYN6C6Y5kEJDUCfwDeBqwCfgMsiIjHqq0zmkmg3FMdL/OqvSexeOVGjnrVXjz49AY2b9vJ2pe28ZYj2/j54+t5Yv3LvPjKDl7cspMXX9nBhi072NHpM43NbOT99IL/yhH7T+2/YpnBJIGx7g46HlgREU8BSLoJOBOomgTG0mFt2R/7vxw+HYDTjn1Vr+VzZ03rs053d/D8pm081fEKK1/cwrTJzfz++c2s3riVzq5uAoiA7gg6u4IdXd10dgdd3d10dgVd3ZHeZ8+71gkaG8Ts/fbguQ1b2NGZlZOW7XqdPbc2NXDAXpNY+9JWdnR191pGWd2BqPTjoNrqlT43qtSuXHfgn1ut9hg3aM3GRGPD6DdvxzoJHASszL1fBZxQXknSQmAhwMEHHzw2kQ1RQ4M4cNpkDpw2uafs9NfOGMeIzMwGbqxnB1VKa31+w0XEVRExLyLmtbW1jUFYZmbFNNZJYBUwK/d+JrBmjGMwM7NkrJPAb4A5kg6V1ALMB+4Y4xjMzCwZ0zGBiOiUdB7wE7IpoosiYtlYxmBmZruM+cliEXEncOdYb9fMzPoq3GUjzMxsFycBM7MCcxIwMyuwMb920GBJ6gCqX+aztunAH0cwnPFSD/tRD/sA3o/dST3sA4zOfhwSEQM6yWq3TwLDIal9oNfP2J3Vw37Uwz6A92N3Ug/7AOO/H+4OMjMrMCcBM7MCq/ckcNV4BzBC6mE/6mEfwPuxO6mHfYBx3o+6HhMwM7Pa6r0lYGZmNTgJmJkVWF0mAUmnSXpc0gpJF453PIMh6RlJj0paLKk9le0r6R5JT6TnfcY7znKSFklaL2lprqxi3Mpcno7PEknHjV/kvVXZj4slrU7HZLGkM3LLLkr78bikU8cn6t4kzZJ0r6TlkpZJ+utUPqGOR439mGjHY5KkhyT9Lu3HP6TyQyU9mI7H99OVlZHUmt6vSMtnj2qAEVFXD7Krkz4JHAa0AL8Djh7vuAYR/zPA9LKyLwMXptcXAv803nFWiPstwHHA0v7iBs4A7iK7ydCJwIPjHX8/+3Ex8KkKdY9O/75agUPTv7vG3WAfZgDHpdd7kt3X++iJdjxq7MdEOx4CpqbXzcCD6e98MzA/lV8JfCK9/ivgyvR6PvD90YyvHlsCPfcxjogdQOk+xhPZmcB16fV1wFnjGEtFEXEfsKGsuFrcZwLXR+YBYJqk3eKenFX2o5ozgZsiYntEPA2sIPv3N64iYm1EPJJebwaWk93adUIdjxr7Uc3uejwiIl5Ob5vTI4C3Aj9I5eXHo3ScfgCcLGnUbjZcj0mg0n2Ma/3D2d0EcLekh9O9lgEOiIi1kP3HAPYft+gGp1rcE/EYnZe6ShbluuN2+/1IXQlvIPv1OWGPR9l+wAQ7HpIaJS0G1gP3kLVSNkZEZ6qSj7VnP9Lyl4D9Riu2ekwCA7qP8W7sTRFxHHA6cK6kt4x3QKNgoh2jK4DDgbnAWuCyVL5b74ekqcAtwPkRsalW1Qplu/N+TLjjERFdETGX7Ja6xwNHVaqWnsd0P+oxCUzo+xhHxJr0vB64jewfzLpS8zw9rx+/CAelWtwT6hhFxLr0n7gbuJpdXQy77X5Iaib74vxeRNyaiifc8ai0HxPxeJRExEbg52RjAtMklW7slY+1Zz/S8r0ZeBfloNVjEpiw9zGWtIekPUuvgbcDS8niPydVOwe4fXwiHLRqcd8BnJ1mpZwIvFTqptgdlfWPv5vsmEC2H/PTbI5DgTnAQ2MdX7nUf3wNsDwivpZbNKGOR7X9mIDHo03StPR6MnAK2fjGvcB7U7Xy41E6Tu80eMKQAAAAzElEQVQF/l+kUeJRMd4j56PxIJvt8AeyfrfPjHc8g4j7MLLZDb8DlpViJ+sP/BnwRHred7xjrRD7jWRN851kv2Q+Wi1usubut9LxeRSYN97x97Mf301xLiH7DzojV/8zaT8eB04f7/hTTH9C1n2wBFicHmdMtONRYz8m2vF4HfDbFO9S4HOp/DCyJLUC+DegNZVPSu9XpOWHjWZ8vmyEmVmB1WN3kJmZDZCTgJlZgTkJmJkVmJOAmVmBOQmYmRWYk4CZWYE5CZiZFdj/BwvO4TvpwB4WAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "graph(loss_history, 'Complete Played Loss History')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHRtJREFUeJzt3XuYXHWd5/H3p7tzg9wMaSDmQogGFRVRs8h4GRivgAzB20xQx7vMOjAz6qoP7Dgsg+POjqOu4y4jiy4j3gIsyhh94uBlQLyBCQiBEIMxAmkDpAnkxiW3/u4f51fdp6ur6lSS6q4+xef1PPVUnXN+XfU9dao//avfOX2OIgIzM+ssXe0uwMzMWs/hbmbWgRzuZmYdyOFuZtaBHO5mZh3I4W5m1oEc7k9Bku6V9Op213GgJJ0qaW2r29qhkbRe0ivaXYcN53AfQ5LeKmm1pF2SHpD0PUkvb3dd9UhaKCkk9RzEz74irecuSY+l59mVuy040OeMiBsj4rmtbnuwJH1N0l5JR43m67STpD5Jp1bNe5+kGyvTEfGsiPhJwfM8U5L/qWYMOdzHiKQPA58D/jtwFLAA+BdgaTvrGi0R8ZOImBoRU4FKyM6szIuI+/PtJXVJKs3nUdI04A3ADuCtY/zaB/zHtuyeiut8qErzy1RmkmYAlwDnRcS3IuKxiNgbEd+JiI+mNpMkfU7S5nT7nKRJadmpqQf1MUlbUq//bElnSLpH0iOS/mvu9S6WdK2kqyXtlHSbpBfUqa1L0gWSfitpq6RrJM1Ki29K99tSb/sP0s+8R9I6SY9Kul7SMQf5vvxU0ick/QJ4DFiQeoXrUt2/lfS+XPtXS7o3N90n6cOS7pS0XdLy3HvWdNu0/EJJD0r6vaT3p28aCxuU/xagn+yP9Tur1qtH0t+m+nekb2tPT8ueL+mHaZs9KOljaf7XJF1csK4flXQn8Hia93FJG9N7tVbSWVV1/LmkX6fld0l6QVrPq6vafUHSpxusa0P53r2kk9PnbYekhyT9U2p2U1pe+eb2n9Jn7yJJ96XP9ZclTU/tnpm2wbsl3Q98P33WPlD12ndLOvNga+9oEeHbKN+A04B9QE+DNpcANwNHAr3Az4FPpGWnpp+/CJgAvJ8sWL4BTCPrGT8JLErtLwb2Am9O7T8C/A6YkJbfC7w6Pf5get15wCTg/wDL07KFQOTrBs4GNgDPAXqAjwM/L1j/Ec+T5v801fKcVGcP8MfAIkDAK4EngBNS+1cD9+Z+vi/VfjRwBHAP8L6DaHsmsDnVcTiwPNW7sME6/Zgs2J8O7AdekFt2IXAHsJisA3UiMAuYATwE/HV6r6cDJ6Wf+Rpwce45atV/a9pOU9K8PwHmpNd4K7ALOCotOwfYBLw4vZfHAfPTz+8Cpqd2E4GH8/VXrWcfcGrVvPcBN9ZqA6wCzkmPpwEvSY+fCUTV85ybtsOxqe23gX/Ntwf+FTgMmJLW8We5n38xsIUGv1dP5VvbC3gq3IC3AQ8WtPktcEZu+nWVX26ycH8C6E7T09IH/yW59rcCZ6fHFwM355Z1AQ8Ar0jT9zIU7uuAV+XaziH7w9BD7XD/HvDequd+HDimwbqNeJ40/6fARQXvy3fJvvHUC7xluenPAv/7INp+hfSHNE0/mwbhnsJoAHhemv4R8Jmqbfn6Gj/3Z8DqOs/ZTLi/o+C9uqvyuqmm8+q0+wHw7vT4bGBNg+fsI/tjsC13e5z64f5zsk7IEVXPUyvcfwycm5t+LrA7faYq4b4gt3xKev1KJ+ZzwOdb9XvaaTcPy4yNrcDsgnHDpwP35abvS/MGnyMi9qfHT6T7h3LLnwCm5qY3VR5ExADZL2D++SqOAa6TtE3SNrKw30+2X6CWY4B/zrV/hKxnOLfBujWyKT8h6UxJt6Rhi23Aa4HZDX7+wdzjxxn+HjTb9ulVdQyrqYZ3AHdGxF1p+uvA2yR1p+n5ZAFfbT7Zt56DVf1evUvSHblt8WyG3qt6NQBcCbw9PX478NWC1z0zImZWbsBfNWj7buB4YL2kX0o6o0HbWp/5iWTfXCvyn+MngGsZeq+XNVH7U5bDfWz8gmzY5OwGbTaTBWfFgjTvYM2vPFC2o3JenefbBJye/+WNiMkR8XuynlOt9n9e1X5KRPz8IOscfA1JU8h+ef+BbHhhJvB9sj8eo+kBsvenYn69hpJEFu7HpTHzB4FPkf0xfF1qtgl4Ro0frzcfsn0Oh+Wmj67RJv9eLQK+AHyArJc8E/g1Q+9Vo9f6FvBiSc8FTicb3muJiFgfEcvIhhc/A3xT0mRqf5Zqfeb3kA05Vp6v+ueuJPsm/Frg0YhY1araO43DfQxExHayr6qXKtsRepikCZJOl/Sp1Gw58HFJvZJmp/ZfO4SXfbGkN6ZvCx8k+7p7c412lwGfrOwUTa9fOYKnn2z4YVFV+wtTMCBphqS3HEKdeZPIem79wP60o+xVLXruRq4B3ivpWZIOA/62QduXk4X/ErKx9BOB56XnqOxY/RLw95KeocyJynZSryDbaXy+pImSpks6Kf3M7cDrJT1N0hwa944h+9YRZO+VlO14fnZu+ZeAj0l6YaphsaT5ABHxOHAd2WfuZ+kPeUtI+jNJs9O3xe2pxgGysfFIf5QqlgMfVnbI7TTgk2T7ewYavMRPyfbP/CPutTfkcB8jEfFZ4MNkOyD7yXpW5wP/lpr8PbAaWAPcCdyW5h2sbwN/CjxKNtb7xojYW6PdP5OFzvcl7ST7A/CSVPPjZL9wP0tf/U+OiOvIfrGukrSDbJz39EOoc1BEbAM+RBY8j5DtEP5uK5674HW/Q9YLvgn4DfCztGh3jebvBK6LiLUR8WDlRvY+LpU0E/gnsu36I7JDJS8HJqc/8q8B3kQWdvcAp6Tn/TLZkNh9wL8DVxXUvAb4PPBLsm8ezwZuyS1fTradrk41fAt4Wu4prgSeT+sD8gxgXfosfRr404jYExE7yb6R3ZI+S0uAL6b6fgJsBHaS7WyuK/Xkv0r2B/XrLa69o2jktx4ru3RI3TMj4u1FbW0kSc8n++M6qaAXWVqpB70GODoidrW7ngMh6T1kO5dPbXct45l77maApDekoZIjgP8BfLuDg72L7FvkN0oY7IcBf0H2bcgacLibZc4jO977N2Q7v89rbzmjQ9k/1O0gO7z279pbzYGR9HqyIc37yYZzrAEPy5iZdSD33M3MOlDbTsYze/bsWLhwYbte3syslG699daHI6K3qF3bwn3hwoWsXr26XS9vZlZKku4rbuVhGTOzjuRwNzPrQA53M7MO5HA3M+tADnczsw5UGO6SrkiXwLqrznJJ+rykDZLWSHpR68s0M7MD0UzP/ctkl4mr53Syy4ktJrts1hcOvSwzMzsUhce5R8RNanyh4KXAV9KpOG+WNFPSnIh4oEU1Nu22+x/lip/+ji07hs7UetSMyZxyXC+33vcoL33GEdzyu62cMHcm197WBwEzDpvAlAndPLjjydqXE0imTOxmUk8X2x6vddbc5k2d3EOXxKQJXQiG1VrP9CkTGIhg15P7Dum1zWx8+INnHMGHXnPcqL5GK/6JaS7DL//Vl+aNCHdJ55L17lmwYEELXnq4T/37r7l54yMj5n/njuwCRMt/eX+ac/+INmZmY+XI6ZNG/TVaEe61LoFWsw8cEZeTTtW5ZMmSlp+x7KHUC/7E2c/juCOnctfmHXziu3fXbLtg1mF86s0n8N01m9nY/xjveulCZkyZUPe5tz2xl937Bjhy2qSDvuZbAI8+toe9A8Eju3YzEPCcOdPpavCEAWzdtYcuwdMOnzjq15szs9F3xNSJo/4arQj3PoZfc7LetTpH3cM7s3D/4xPmMPOwiSxZOIvP/fAedtYYznjls4/k5EVHcPKiI8a6TDOzUdeKQyFXAO9IR82cDGxvx3j7k3v3s3P3Pnq6NNgD7+4S3/zAS/nPp4y8TvCZJ8wZ6xLNzMZMYc9d0nKyE/vPltQH/DeyC9QSEZcBK8mum7gBeBx492gV28jWx/YA2ded7AL1meOOmsZHXnscl/34twC87SULeP3z57Bk4ax2lGlmNiaaOVrmnILlwTi4ak1lSGb21JE7Knq6u3j6jMls3v4kb1kynxPnzxzr8szMxlTbTvnbalsfqx/uANed9zL6Hn3cwW5mTwkdE+4P7xwalqnlqOmTOWr65LEsycysbTrm3DL9u7Kee2+dnruZ2VNJx4T71l1Zz73esIyZ2VNJx4T7w6nnPnva6P9zgJnZeNcx4f5IOhRy1uHuuZuZdUy4P7Yn+y/UqZO621yJmVn7dUy4P7l3AIDJExzuZmYdFO77AYe7mRl0YLhPcbibmXVOuD/hnruZ2aCOCXf33M3MhnREuEfE4A7VST0dsUpmZoekI5Jw974s2Cf2dNHV6LJGZmZPER0R7k/s8ZCMmVleR4T7k/sqO1M7YnXMzA5ZR6She+5mZsN1RLj7v1PNzIbrjHDf52PczczyOiPc93jM3cwsryPSsNJz95i7mVmmI8L9iT0eczczy+uIcPepB8zMhuuIcK+cNGySw93MDOiQcHfP3cxsuI4Kdx8tY2aW6Yg0rPwTk3vuZmaZjgh3X6jDzGy4jgj3wWGZiQ53MzPokHAf7Ln7Qh1mZkCT4S7pNEnrJW2QdEGN5cdI+pGkNZJulDSv9aXWt7sy5u6eu5kZ0ES4S+oGLgVOB44HzpF0fFWzTwNfiYgTgEuAf2h1oY0M9dwd7mZm0FzP/SRgQ0RsjIg9wFXA0qo2xwM/So9vqLF8VA2ez909dzMzoLlwnwtsyk33pXl5dwBvSo/fAEyTdET1E0k6V9JqSav7+/sPpt6atj+xF4Dpkye07DnNzMqsmXCvdcXpqJr+CHCKpF8BpwC/B/aN+KGIyyNiSUQs6e3tPeBi66mE+4wpDnczM4CeJtr0AfNz0/OAzfkGEbEZeCOApKnAmyJie6uKLLLD4W5mNkwzPfdVwGJJx0qaCCwDVuQbSJotqfJcFwJXtLbM+vYPBDt370OCaZOb+VtlZtb5CsM9IvYB5wPXA+uAayJiraRLJJ2Vmp0KrJd0D3AU8MlRqneESq992qQeurpqjSCZmT31NNXVjYiVwMqqeRflHl8LXNva0pqz48m0M9VDMmZmg0r/L53emWpmNpLD3cysAznczcw6kMPdzKwDdUy4e4eqmdmQjgl399zNzIaUPtx3uOduZjZCB4R7dgob99zNzIaUPtwffXwP4HA3M8srfbhv2bkbgCOnTWpzJWZm40fpw/2h7U8CcPT0yW2uxMxs/Ch1uD+2ex87d+9jYk8XMw/zsIyZWUWpw/3BHUO9dslnhDQzqyh1uHtIxsystlKHe6XnftQMh7uZWV5nhLuPlDEzG6bU4T44LOOeu5nZMKUO98Geu8fczcyGKXW4P7Qj+wcm99zNzIYrebj7aBkzs1pKG+77B2Lo1APTvUPVzCyvtOG+dddu9g8Esw6fyKSe7naXY2Y2rpQ23L0z1cysvvKG++B/p3pIxsysWmnDfXBnqo+UMTMbobThXhmWOXKaw93MrFp5w327j3E3M6unlOE+MBCs6dsGwNyZU9pcjZnZ+FPKcL9h/RZ+s2UXR0+fzEsWzWp3OWZm405T4S7pNEnrJW2QdEGN5Qsk3SDpV5LWSDqj9aUO+eZtfQC85+ULfYy7mVkNheEuqRu4FDgdOB44R9LxVc0+DlwTES8ElgH/0upC83bt3g/As46ePpovY2ZWWs303E8CNkTExojYA1wFLK1qE0AlaWcAm1tX4kgRAUCXr6xnZlZTM+E+F9iUm+5L8/IuBt4uqQ9YCfxlrSeSdK6k1ZJW9/f3H0S5mf0DlXB3upuZ1dJMuNdK0KiaPgf4ckTMA84AvippxHNHxOURsSQilvT29h54tclA6rk7283Mamsm3PuA+bnpeYwcdnkvcA1ARPwCmAzMbkWBtaSOu3vuZmZ1NBPuq4DFko6VNJFsh+mKqjb3A68CkPQcsnA/+HGXAkNj7g53M7NaCsM9IvYB5wPXA+vIjopZK+kSSWelZv8FeL+kO4DlwLuiksCjYKjnPlqvYGZWbj3NNIqIlWQ7SvPzLso9vht4WWtLq29ozN3pbmZWSyn/Q9U9dzOzxkoZ7h5zNzNrrJThPuBwNzNrqJzhPpDdd5WyejOz0VfKeHTP3cyssVKGe/ifmMzMGipluA/4xGFmZg2VOtx9nLuZWW2lDPfwce5mZg2VMtz3e4eqmVlDpQx3Hy1jZtZYOcM9HefubDczq62U4T54+gEPupuZ1VTKcPeJw8zMGitpuHvM3cyskZKGe3bvbDczq62U4e5T/pqZNVbKcPewjJlZYyUN9+zeO1TNzGorabj7UEgzs0ZKGe4+5a+ZWWOlDHef8tfMrLGSh7vT3cyslpKGe3bvbDczq62c4T7gnruZWSPlDHcPy5iZNVTScM/uvUPVzKy20oV75dQD4GuompnVU7pwd6/dzKxYCcPd4+1mZkWaCndJp0laL2mDpAtqLP+fkm5Pt3skbWt9qRmHu5lZsZ6iBpK6gUuB1wB9wCpJKyLi7kqbiPhQrv1fAi8chVrTa1VeZ7Rewcys/JrpuZ8EbIiIjRGxB7gKWNqg/TnA8lYUV4t77mZmxZoJ97nAptx0X5o3gqRjgGOB/6iz/FxJqyWt7u/vP9BaAe9QNTNrRjPhXitGo8Y8gGXAtRGxv9bCiLg8IpZExJLe3t5maxzGPXczs2LNhHsfMD83PQ/YXKftMkZxSAYgBrJ7n8vdzKy+ZsJ9FbBY0rGSJpIF+IrqRpKeBTwN+EVrSxzOp/s1MytWGO4RsQ84H7geWAdcExFrJV0i6axc03OAqyL/L6SjwMMyZmbFCg+FBIiIlcDKqnkXVU1f3Lqy6hs63a/D3cysnhL/h2qbCzEzG8dKHO5OdzOzekoY7tm9e+5mZvWVL9xTunvM3cysvtKFe+VYnK7SVW5mNnZKF5EeczczK+ZwNzPrQCUM9+ze2W5mVl/pwj3cczczK1S6cPehkGZmxUoY7u65m5kVKW24+zh3M7P6ShfulePcu0tXuZnZ2CldRHpYxsysWAnDPbv3sIyZWX2lC/f9Az7lr5lZkdKFu49zNzMrVrpw93HuZmbFShjuPhTSzKxIacPdPXczs/pKF+6D53N3z93MrK7ShbuPczczK1bCcM/une1mZvWVMNzdczczK1K6cA/vUDUzK1S6cB8YyO7dczczq6984e7j3M3MCpUw3LN7D8uYmdVXunCvjLl3O93NzOoqXbjv99EyZmaFmgp3SadJWi9pg6QL6rT5E0l3S1or6RutLXOIj3M3MyvWU9RAUjdwKfAaoA9YJWlFRNyda7MYuBB4WUQ8KunI0SrYp/w1MyvWTM/9JGBDRGyMiD3AVcDSqjbvBy6NiEcBImJLa8sc4hOHmZkVaybc5wKbctN9aV7eccBxkn4m6WZJp9V6IknnSlotaXV/f/9BFezj3M3MijUT7rVSNKqme4DFwKnAOcCXJM0c8UMRl0fEkohY0tvbe6C1Aj7O3cysGc2Eex8wPzc9D9hco823I2JvRPwOWE8W9i0XPs7dzKxQM+G+Clgs6VhJE4FlwIqqNv8G/BGApNlkwzQbW1lohU8cZmZWrDDcI2IfcD5wPbAOuCYi1kq6RNJZqdn1wFZJdwM3AB+NiK2jUfDgf6iW7gh9M7OxU3goJEBErARWVs27KPc4gA+n26jymLuZWbHS9X99yl8zs2KlC/cBX0PVzKxQCcPdO1TNzIqUMNyze2e7mVl9pQv3wVP+Ot3NzOoqXbjvT133Lu9RNTOrq3Th7mEZM7NiJQx371A1MytSunD3ce5mZsVKF+4+zt3MrFgJw92nHzAzK1LCcM/uPSxjZlZf6cLd11A1MytWunD3NVTNzIqVMNyze4+5m5nVV8Jw97CMmVmR0oW7r6FqZlasdOE+MOCeu5lZkfKFu88tY2ZWqITh7p67mVmR0oZ7twfdzczqKm24O9vNzOorYbhn9z7O3cysvtKFu08/YGZWrHThPjCQ3XtYxsysvvKFu3vuZmaFShju2b2z3cysvtKFu8fczcyKlS7cB4dlSle5mdnYaSoiJZ0mab2kDZIuqLH8XZL6Jd2ebu9rfakZX0PVzKxYT1EDSd3ApcBrgD5glaQVEXF3VdOrI+L8UahxGF9D1cysWDM995OADRGxMSL2AFcBS0e3rPp8yl8zs2LNhPtcYFNuui/Nq/YmSWskXStpfq0nknSupNWSVvf39x9EuT4U0sysGc2Ee60Ujarp7wALI+IE4IfAlbWeKCIuj4glEbGkt7f3wCpNfG4ZM7NizYR7H5Dvic8DNucbRMTWiNidJr8IvLg15Y3kc8uYmRVrJtxXAYslHStpIrAMWJFvIGlObvIsYF3rShzOV2IyMytWeLRMROyTdD5wPdANXBERayVdAqyOiBXAX0k6C9gHPAK8a7QKHjqf+2i9gplZ+RWGO0BErARWVs27KPf4QuDC1pZW29TJE5h1+EQmdnePxcuZmZVSU+E+nvyvc17Y7hLMzMY9D26YmXUgh7uZWQdyuJuZdSCHu5lZB3K4m5l1IIe7mVkHcribmXUgh7uZWQdS5ZqkY/7CUj9w30H++Gzg4RaW0y5ej/GjE9YBvB7jzWisxzERUXha3baF+6GQtDoilrS7jkPl9Rg/OmEdwOsx3rRzPTwsY2bWgRzuZmYdqKzhfnm7C2gRr8f40QnrAF6P8aZt61HKMXczM2usrD13MzNrwOFuZtaBShfukk6TtF7SBkkXtLueAyHpXkl3Srpd0uo0b5akH0j6Tbp/WrvrzJN0haQtku7KzatZszKfT9tmjaQXta/y4eqsx8WSfp+2x+2SzsgtuzCtx3pJr2tP1SNJmi/pBknrJK2V9Ndpfmm2SYN1KNX2kDRZ0i8l3ZHW4+/S/GMl3ZK2xdXp2tNImpSmN6TlC0e1wIgozY3sGq6/BRYBE4E7gOPbXdcB1H8vMLtq3qeAC9LjC4B/bHedVfX9IfAi4K6imoEzgO8BAk4Gbml3/QXrcTHwkRptj0+frUnAsekz193udUi1zQFelB5PA+5J9ZZmmzRYh1Jtj/SeTk2PJwC3pPf4GmBZmn8Z8IH0+C+Ay9LjZcDVo1lf2XruJwEbImJjROwBrgKWtrmmQ7UUuDI9vhI4u421jBARN5Fd9DyvXs1Lga9E5mZgpqQ5Y1NpY3XWo56lwFURsTsifgdsIPvstV1EPBARt6XHO4F1wFxKtE0arEM943J7pPd0V5qckG4BvBK4Ns2v3haVbXQt8CpJGq36yhbuc4FNuek+Gn8oxpsAvi/pVknnpnlHRcQDkH3ogSPbVl3z6tVcxu1zfhquuCI3JFaK9Uhf619I1mMs5TapWgco2faQ1C3pdmAL8AOybxXbImJfapKvdXA90vLtwBGjVVvZwr3WX7kyHcv5soh4EXA6cJ6kP2x3QS1Wtu3zBeAZwInAA8Bn0vxxvx6SpgLfBD4YETsaNa0xb1ysS411KN32iIj9EXEiMI/s28RzajVL92O6HmUL9z5gfm56HrC5TbUcsIjYnO63ANeRfRgeqnxNTvdb2ldh0+rVXKrtExEPpV/OAeCLDH3VH9frIWkCWSh+PSK+lWaXapvUWoeybg+AiNgG3Eg25j5TUk9alK91cD3S8hk0P1R4wMoW7quAxWlv9ESynRIr2lxTUyQdLmla5THwWuAusvrfmZq9E/h2eyo8IPVqXgG8Ix2hcTKwvTJUMB5VjT2/gWx7QLYey9LRDccCi4FfjnV9taQx2v8LrIuIz+YWlWab1FuHsm0PSb2SZqbHU4BXk+0/uAF4c2pWvS0q2+jNwH9E2rs6Ktq9x/lAb2R7/+8hG9v6m3bXcwB1LyLb438HsLZSO9mY24+A36T7We2utaru5WRfkfeS9TzeW69msq+dl6ZtcyewpN31F6zHV1Oda8h+8ebk2v9NWo/1wOntrj9X18vJvsqvAW5PtzPKtE0arEOptgdwAvCrVO9dwEVp/iKyPz4bgP8HTErzJ6fpDWn5otGsz6cfMDPrQGUbljEzsyY43M3MOpDD3cysAznczcw6kMPdzKwDOdzNzDqQw93MrAP9f5Wy3x09DiP6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "graph(train_accuracy_history, 'Complete Training Accuracy History')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions, labels = generate_outputs(played_test_data, binary_played_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7163593566104355"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy(predictions, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('./data/complete_played_itemGamma', itemGamma)\n",
    "np.save('./data/complete_played_itemBiases', itemBiases)\n",
    "np.save('./data/complete_played_userGamma', userGamma)\n",
    "np.save('./data/complete_played_loss', loss_history)\n",
    "np.save('./data/complete_played_train_accuracy', train_accuracy_history)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}