p53-finished.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting environmental carcinogens with logistic regression, knn, gradient boosting and molecular fingerprinting\n",
    "\n",
    "### Balancing imbalanced data, exploring accuracy metrics, and an introduction to cheminformatics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing the data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# general and data handling\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from collections import Counter\n",
    "\n",
    "# Required RDKit modules\n",
    "import rdkit as rd\n",
    "from rdkit.Chem.Fingerprints import FingerprintMols\n",
    "from rdkit import RDConfig\n",
    "from rdkit.Chem import PandasTools\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem.Draw import IPythonConsole\n",
    "from rdkit.Chem import rdFingerprintGenerator\n",
    "from rdkit import DataStructs\n",
    "from rdkit.Chem import AllChem as Chem\n",
    "from rdkit.Chem.rdMolDescriptors import GetAtomPairFingerprint\n",
    "from rdkit.Chem.AtomPairs import Torsions\n",
    "\n",
    "# modeling\n",
    "import sklearn as sk\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import validation_curve\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from imblearn.over_sampling import ADASYN\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "# Graphing\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading and formatting\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Active</th>\n",
       "      <th>ID</th>\n",
       "      <th>SMILES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NCGC00166288-01</td>\n",
       "      <td>CCN1C(=CC=Cc2sc3ccccc3[n+]2CC)Sc2ccccc21.[I-]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>NCGC00185752-01</td>\n",
       "      <td>COC(=O)CC(O)(CCCC(C)(C)O)C(=O)OC1C(OC)=CC23CCCN2CCc2cc4c(cc2C13)OCO4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>NCGC00094121-01</td>\n",
       "      <td>Cc1cccc(C)c1OCC(C)N.Cl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>NCGC00241107-01</td>\n",
       "      <td>CO.COc1cc(Nc2c(C#N)cnc3cc(OCCCN4CCN(C)CC4)c(OC)cc23)c(Cl)cc1Cl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>NCGC00094586-01</td>\n",
       "      <td>CC1(C)SC2C(NC(=O)C(N)c3ccc(O)cc3)C(=O)N2C1C(=O)O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Active               ID                                             SMILES\n",
       "0      1  NCGC00166288-01      CCN1C(=CC=Cc2sc3ccccc3[n+]2CC)Sc2ccccc21.[I-]\n",
       "1      1  NCGC00185752-01  COC(=O)CC(O)(CCCC(C)(C)O)C(=O)OC1C(OC)=CC23CCC...\n",
       "2      0  NCGC00094121-01                             Cc1cccc(C)c1OCC(C)N.Cl\n",
       "3      1  NCGC00241107-01  CO.COc1cc(Nc2c(C#N)cnc3cc(OCCCN4CCN(C)CC4)c(OC...\n",
       "4      0  NCGC00094586-01   CC1(C)SC2C(NC(=O)C(N)c3ccc(O)cc3)C(=O)N2C1C(=O)O"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdfFile = os.path.join(RDConfig.RDDataDir, r'C:/Users/gurka/Projects/p53/sr-p53.sdf')\n",
    "p53 = PandasTools.LoadSDF(sdfFile,smilesName='SMILES',molColName='Molecule', includeFingerprints=True)\n",
    "p53 = p53.drop(['DSSTox_CID', 'FW', \"Formula\", 'Molecule'], axis=1)\n",
    "p53.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## mol generation and unique records\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "p53[\"mol\"] = [Chem.MolFromSmiles(x) for x in p53[\"SMILES\"]]\n",
    "#frame1['MFP2BV'] = [Chem.GetMorganFingerprintAsBitVect(x, 2, nBits=1024) for x in frame1[\"mol\"]]\n",
    "#frame1[\"bitstring\"] = [x.ToBitString for x in frame1[\"MFP2BV\"]]\n",
    "\n",
    "#count the nummber of unique rows in the SMILES column\n",
    "p53['SMILES'].nunique()\n",
    "\n",
    "#Compare that with total rows\n",
    "p53['SMILES'].count()\n",
    "\n",
    "#we can drop the duplicated ones\n",
    "p53 = p53.drop_duplicates(['SMILES'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fingerprint generation\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a column for the morgan fingerprints\n",
    "p53[\"morg_fp\"] = [Chem.GetMorganFingerprintAsBitVect(m, 2, nBits = 2048) for m in p53['mol']]\n",
    "\n",
    "# generate morgan fingeprints with radius 2 contained in a list\n",
    "morg_fp = [Chem.GetMorganFingerprintAsBitVect(m, 2, nBits = 2048) for m in p53['mol']]\n",
    "\n",
    "# convert the RDKit explicit vectors into numpy arrays\n",
    "morg_fp_np = []\n",
    "for fp in morg_fp:\n",
    "  arr = np.zeros((1,))\n",
    "  DataStructs.ConvertToNumpyArray(fp, arr)\n",
    "  morg_fp_np.append(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a column for the RDKit fingerprints\n",
    "p53[\"rd_fp\"] = [Chem.RDKFingerprint(m) for m in p53[\"mol\"]]\n",
    "\n",
    "#The fingerprinting algorithm used is similar to that used in the Daylight fingerprinter\n",
    "rd_fp = [Chem.RDKFingerprint(m) for m in p53[\"mol\"]]\n",
    "\n",
    "rd_fp_np = []\n",
    "for fp in rd_fp:\n",
    "  arr = np.zeros((1,))\n",
    "  DataStructs.ConvertToNumpyArray(fp, arr)\n",
    "  rd_fp_np.append(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a column for the Atom-pair fingerprints\n",
    "p53[\"AP_fp\"] = [Chem.GetHashedAtomPairFingerprintAsBitVect(m) for m in p53[\"mol\"]]\n",
    "\n",
    "#get atom-par fingerprints\n",
    "AP_fp = [Chem.GetHashedAtomPairFingerprintAsBitVect(m) for m in p53[\"mol\"]]\n",
    "\n",
    "#convert the RDKit explicit vectors into numpy arrays\n",
    "AP_fp_np = []\n",
    "for fp in AP_fp:\n",
    "  arr = np.zeros((1,))\n",
    "  DataStructs.ConvertToNumpyArray(fp, arr)\n",
    "  AP_fp_np.append(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a column for the topological torsion fingerprints\n",
    "p53[\"torsion_fp\"] = torsion_fp = [Chem.GetHashedTopologicalTorsionFingerprintAsBitVect(m) for m in p53[\"mol\"]]\n",
    "\n",
    "#get topological torsion fingerprints\n",
    "torsion_fp = [Chem.GetHashedTopologicalTorsionFingerprintAsBitVect(m) for m in p53[\"mol\"]]\n",
    "\n",
    "#convert the RDKit explicit vectors into numpy arrays\n",
    "torsion_fp_np = []\n",
    "for fp in torsion_fp:\n",
    "  arr = np.zeros((1,))\n",
    "  DataStructs.ConvertToNumpyArray(fp, arr)\n",
    "  torsion_fp_np.append(arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setting the x and y variables:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#morgan fingerprints\n",
    "x_morg = morg_fp_np\n",
    "\n",
    "#daylight-like fingerprints\n",
    "x_rd = rd_fp_np\n",
    "\n",
    "#atom-pair fingerprints\n",
    "x_AP = AP_fp_np\n",
    "\n",
    "#topological torsion fingerprints\n",
    "x_torsion = torsion_fp_np\n",
    "\n",
    "#classification labels\n",
    "y = p53.Active.values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sampling "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0.98, '0: 6406, 1: 420')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAFkCAYAAAAAOdHpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGOpJREFUeJzt3X2wHXWd5/H3RwjyGAgaMpgE4kNwhNkV8BbgqsMqLgRn1rAKEp6MDFOxXKRmqlhH2MViFob1oUZRHMVBQYOwIgUysC6jcwtxXdZRiCsiBJAAQrIwJJCAPEQo8Lt/3E68xHuTQ7wn9/5y36+qU6f727/u/nYq+aSrT58+qSokSe142Xg3IEl6aQxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW2Miye5JrknydJIHkhz/Etd/TZJvJ3kyyaNJPjXCmLlJfp3ksg3qx3f7fDrJPyTZfYPlC5Lc2S2/N8nbeuzp7UluTPJEkl++lOPZYDsLk1SSPx9W+0iS27vjvT/JRzZYZ06372eS3JXknZu7f219DG6NlS8AzwEzgBOAC5Ps18uKSbYDBoHvAX8AzAIuG2HoF4BbNlh3P+DvgZO6fT8DfHHY8n8HfBI4GdgF+GPgvh6P6WngEuAjmxo4miTTgDOBOzZcBLwfmAbMAz6cZMGw5d8Afgq8AvgvwFVJpm9uH9q6xG9O6veVZCdgDfBHVfWLrvZ14P9V1Rk9rL8IOKmqRj0T7kLtPcBS4HVVdWJX/2/AnKo6vpt/LXAn8IqqejLJD4GLq+ri3+P43gl8parmbMa6XwJuA94HXFZVXxll3AUM/Xs8Lck+wM+BV1bVk93y/w1cXlVf2szD0FbEM26NhX2AF9aFdudnwH4ASfZK8niSvUZZ/xDgl0n+sbtM8v0k/2rdwiRTgXOA00dYd79uXwBU1b0Mnfnvk2QbYACYnmRZkhVJ/i7JDr/Hsa7XXdoZ9T+mJAd1+99o2CYJ8DZ+e1a+H3DfutDurP/zlAxujYWdgSc2qD3B0KUJqurBqtqtqh4cZf1ZwALgAuBVwP8Eru0uoQCcy9BZ8/KXuO8ZwBTgaIaCcX/gAOCs3g9tdFX1p1X1iZGWdf9pfBE4rap+s4lN/TVD/xa/2s1v9M9TMrg1Fp4Cpm5Qmwo8OcLYkawFbqqqf6yq54C/Zeja7huS7A+8Ezh/M/a9tpv/fFU9XFWPAp8B3tVjX7+P/wjcVlX/vLFBST7M0LXuP6mqZ7vy7/vnqa1dVW11ryOOOKK05Tz11FM1ZcqU+sUvfrG+dtJJJ9VHP/rRntY/66yz6u1vf/v6+d/85jc1derUuvXWW+v888+vHXfcsWbMmFEzZsyonXbaqbbffvs64IADqqrqzDPPrOOPP379uvfee29NmTKlfvWrX1VV1axZs2rx4sXrl1911VW1//77v6TjGxwcrL333vslrTN//vzabbfd1vc9ZcqUmjp1ap166qnrx1x88cU1c+bMuvfee1+07t13310vf/nL1x9DVdXb3va2uvDCC19SD2pSTxk37iHbj9eb3vSmsfgD1Etw7LHH1oIFC+qpp56qm266qaZOnVq33357T+veddddtcMOO9Tg4GA9//zz9ZnPfKZe85rX1LPPPltPP/10Pfzww+tfp59+er33ve+tlStXVlXV7bffXrvsskv94Ac/qKeeeqpOOOGEOvbYY9dv+2Mf+1gNDAzUI488UqtXr663vvWtddZZZ61fDtSNN944Yl8vvPBCrV27tq6//vraa6+9au3atfXss8/2dExr1qx5Ud9vfvOb69Of/nQ9/vjjVVV12WWX1YwZM2rp0qUjrn/wwQfX6aefXmvXrq1vfetbteuuu64/Zm3VDG5tOY899ljNnz+/dtxxx5o9e3Zdfvnl65c98MADtdNOO9UDDzww6vpXX311vfa1r61ddtmlDj300FFD/+yzz64TTjjhRbXLL7+8Zs+eXTvuuGO9+93vrscee2z9sueee64+9KEP1a677lozZsyo0047rdauXVtVVcuXL6+dd965Hn300RH3deONNxbwotehhx66fvm8efPqvPPO2+SfTVXVoYceWl/+8pfXz8+ZM6e23Xbb2mmnnda/PvjBD65ffv/999ehhx5a22+/fe2zzz41ODjY037UvJ4ybqu8HXBgYKCWLFky3m1ogrvsssu44447+PjHPz7erUjrpJdB2/a7C2miOvHEE8e7BWmzeFeJJDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xm9ObuDBO3803i1oC9jrDYeMdwvSZvOMW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1pq/BnWS3JFcluSvJnUnenGT3JINJ7unep3Vjk+SCJMuS3JbkwGHbWdiNvyfJwn72LEkTXb/PuD8HfKeq/hB4I3AncAZwQ1XNBW7o5gGOBOZ2r0XAhQBJdgfOBg4GDgLOXhf2kjQZ9S24k0wF/hi4GKCqnquqx4H5wOJu2GLgqG56PnBpDfkRsFuSPYEjgMGqWl1Va4BBYF6/+pakia6fZ9yvAVYBX03y0yRfSbITMKOqHgbo3vfoxs8Elg9bf0VXG60uSZNSP4N7W+BA4MKqOgB4mt9eFhlJRqjVRuovXjlZlGRJkiWrVq3anH4lqQn9DO4VwIqq+nE3fxVDQf5IdwmE7n3lsPGzh60/C3hoI/UXqaqLqmqgqgamT58+pgciSRNJ34K7qv4FWJ7k9V3pMGApcB2w7s6QhcC13fR1wPu7u0sOAZ7oLqV8Fzg8ybTuQ8nDu5okTUr9/pX304DLk2wH3AeczNB/FlcmOQV4EDimG3s98C5gGfBMN5aqWp3kXOCWbtw5VbW6z31L0oTV1+CuqluBgREWHTbC2AJOHWU7lwCXjG13ktQmvzkpSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMb0NbiT/DLJz5PcmmRJV9s9yWCSe7r3aV09SS5IsizJbUkOHLadhd34e5Is7GfPkjTRbYkz7rdX1f5VNdDNnwHcUFVzgRu6eYAjgbndaxFwIQwFPXA2cDBwEHD2urCXpMloPC6VzAcWd9OLgaOG1S+tIT8CdkuyJ3AEMFhVq6tqDTAIzNvSTUvSRNHv4C7gn5L8JMmirjajqh4G6N736OozgeXD1l3R1UarS9KktG2ft/+WqnooyR7AYJK7NjI2I9RqI/UXrzz0H8MigL322mtzepWkJvT1jLuqHureVwLXMHSN+pHuEgjd+8pu+Apg9rDVZwEPbaS+4b4uqqqBqhqYPn36WB+KJE0YfQvuJDsl2WXdNHA4cDtwHbDuzpCFwLXd9HXA+7u7Sw4BnugupXwXODzJtO5DycO7miRNSv28VDIDuCbJuv3896r6TpJbgCuTnAI8CBzTjb8eeBewDHgGOBmgqlYnORe4pRt3TlWt7mPfkjSh9S24q+o+4I0j1B8DDhuhXsCpo2zrEuCSse5RklrkNyclqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1Ji+B3eSbZL8NMm3u/lXJ/lxknuSfDPJdl395d38sm75nGHbOLOr353kiH73LEkT2ZY44/4L4M5h858Ezq+qucAa4JSufgqwpqpeB5zfjSPJvsACYD9gHvDFJNtsgb4laULqa3AnmQX8CfCVbj7AO4CruiGLgaO66fndPN3yw7rx84ErqurZqrofWAYc1M++JWki6/cZ92eBvwJ+082/Ani8qp7v5lcAM7vpmcBygG75E9349fUR1pGkSadvwZ3kT4GVVfWT4eURhtYmlm1sneH7W5RkSZIlq1atesn9SlIr+nnG/Rbg3Ul+CVzB0CWSzwK7Jdm2GzMLeKibXgHMBuiW7wqsHl4fYZ31quqiqhqoqoHp06eP/dFI0gTRt+CuqjOralZVzWHow8XvVdUJwI3A0d2whcC13fR13Tzd8u9VVXX1Bd1dJ68G5gI396tvSZrott30kDH3UeCKJH8D/BS4uKtfDHw9yTKGzrQXAFTVHUmuBJYCzwOnVtULW75tSZoYtkhwV9X3ge930/cxwl0hVfVr4JhR1j8POK9/HUpSO/zmpCQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5Ia01NwJ7mhl5okqf82+nTAJNsDOwKvTDKN3/4azVTgVX3uTZI0gk091vWDwF8yFNI/4bfB/SvgC33sS5I0io0Gd1V9DvhcktOq6vNbqCdJ0kb09EMKVfX5JP8GmDN8naq6tE99SZJG0VNwJ/k68FrgVmDdz4YVYHBL0hbW60+XDQD7dj/eK0kaR73ex3078Af9bESS1Jtez7hfCSxNcjPw7LpiVb27L11JkkbVa3D/dT+bkCT1rte7Sv5XvxuRJPWm17tKnmToLhKA7YApwNNVNbVfjUmSRtbrGfcuw+eTHAUc1JeOJEkbtVlPB6yqfwDeMca9SJJ60OulkvcMm30ZQ/d1e0+3JI2DXu8q+ffDpp8HfgnMH/NuJEmb1Os17pP73YgkqTe9/pDCrCTXJFmZ5JEkVyeZ1e/mJEm/q9cPJ78KXMfQc7lnAv+jq0mStrBeg3t6VX21qp7vXl8DpvexL0nSKHoN7keTnJhkm+51IvBYPxuTJI2s1+D+M+B9wL8ADwNHA35gKUnjoNfbAc8FFlbVGoAkuwN/y1CgS5K2oF7PuP/1utAGqKrVwAH9aUmStDG9BvfLkkxbN9Odcfd6ti5JGkO9BvengR8mOTfJOcAPgU9tbIUk2ye5OcnPktyR5L929Vcn+XGSe5J8M8l2Xf3l3fyybvmcYds6s6vfneSIzTlQSdpa9BTc3a+5vxd4BFgFvKeqvr6J1Z4F3lFVbwT2B+YlOQT4JHB+Vc0F1gCndONPAdZU1euA87txJNkXWADsB8wDvphkm94PUZK2Lj0/HbCqllbV31XV56tqaQ/jq6qe6mandK9i6KmCV3X1xcBR3fT8bp5u+WFJ0tWvqKpnq+p+YBk+UlbSJLZZj3XtVXfP963ASmAQuBd4vKqe74asYOibmHTvywG65U8ArxheH2Gd4ftalGRJkiWrVq3qx+FI0oTQ1+Cuqheqan9gFkNnyW8YaVj3nlGWjVbfcF8XVdVAVQ1Mn+6XOiVtvfoa3OtU1ePA94FDgN2SrLsjZRbwUDe9ApgN0C3fFVg9vD7COpI06fQtuJNMT7JbN70D8E7gTuBGhr55CbAQuLabvq6bp1v+vaqqrr6gu+vk1cBc4OZ+9S1JE10/78XeE1jc3QHyMuDKqvp2kqXAFUn+BvgpcHE3/mLg60mWMXSmvQCgqu5IciWwlKEfcTi1ql7oY9+SNKH1Lbir6jZG+HZlVd3HCHeFVNWvgWNG2dZ5wHlj3aMktWiLXOOWJI0dg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWpM34I7yewkNya5M8kdSf6iq++eZDDJPd37tK6eJBckWZbktiQHDtvWwm78PUkW9qtnSWpBP8+4nwdOr6o3AIcApybZFzgDuKGq5gI3dPMARwJzu9ci4EIYCnrgbOBg4CDg7HVhL0mTUd+Cu6oerqr/200/CdwJzATmA4u7YYuBo7rp+cClNeRHwG5J9gSOAAaranVVrQEGgXn96luSJrotco07yRzgAODHwIyqehiGwh3Yoxs2E1g+bLUVXW20+ob7WJRkSZIlq1atGutDkKQJo+/BnWRn4GrgL6vqVxsbOkKtNlJ/caHqoqoaqKqB6dOnb16zktSAvgZ3kikMhfblVfWtrvxIdwmE7n1lV18BzB62+izgoY3UJWlS6uddJQEuBu6sqs8MW3QdsO7OkIXAtcPq7+/uLjkEeKK7lPJd4PAk07oPJQ/vapI0KW3bx22/BTgJ+HmSW7vafwY+AVyZ5BTgQeCYbtn1wLuAZcAzwMkAVbU6ybnALd24c6pqdR/7lqQJrW/BXVU3MfL1aYDDRhhfwKmjbOsS4JKx606S2uU3JyWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUmL4Fd5JLkqxMcvuw2u5JBpPc071P6+pJckGSZUluS3LgsHUWduPvSbKwX/1KUiv6ecb9NWDeBrUzgBuqai5wQzcPcCQwt3stAi6EoaAHzgYOBg4Czl4X9pI0WfUtuKvqB8DqDcrzgcXd9GLgqGH1S2vIj4DdkuwJHAEMVtXqqloDDPK7/xlI0qSypa9xz6iqhwG69z26+kxg+bBxK7raaHVJmrQmyoeTGaFWG6n/7gaSRUmWJFmyatWqMW1OkiaSLR3cj3SXQOjeV3b1FcDsYeNmAQ9tpP47quqiqhqoqoHp06ePeeOSNFFs6eC+Dlh3Z8hC4Nph9fd3d5ccAjzRXUr5LnB4kmndh5KHdzVJmrS27deGk3wD+LfAK5OsYOjukE8AVyY5BXgQOKYbfj3wLmAZ8AxwMkBVrU5yLnBLN+6cqtrwA09JmlT6FtxVddwoiw4bYWwBp46ynUuAS8awNUlq2kT5cFKS1CODW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1Jjenbr7xL+l0Dew+MdwvqsyUPLOn7PjzjlqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNaSa4k8xLcneSZUnOGO9+JGm8NBHcSbYBvgAcCewLHJdk3/HtSpLGRxPBDRwELKuq+6rqOeAKYP449yRJ46KV4J4JLB82v6KrSdKk08pPl2WEWr1oQLIIWNTNPpXk7r53tfV4JfDoeDehrdKk+7uVjBRXPftOVc3b1KBWgnsFMHvY/CzgoeEDquoi4KIt2dTWIsmSqvLHEDXm/LvVH61cKrkFmJvk1Um2AxYA141zT5I0Lpo4466q55N8GPgusA1wSVXdMc5tSdK4aCK4AarqeuD68e5jK+UlJvWLf7f6IFW16VGSpAmjlWvckqSOwT3J+SgB9UOSS5KsTHL7ePeyNTK4JzEfJaA++hqwyfuRtXkM7snNRwmoL6rqB8Dq8e5ja2VwT24+SkBqkME9uW3yUQKSJh6De3Lb5KMEJE08Bvfk5qMEpAYZ3JNYVT0PrHuUwJ3AlT5KQGMhyTeAfwZen2RFklPGu6etid+clKTGeMYtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uTVpL/kKSS/OEmxn0gyauGzX/Fh3FpPBncmsyOA25i6ItHG/MBYH1wV9WfV9XSPvYlbZTBrUkpyc7AW4BTGBbcSf4qyc+T/CzJJ5IcDQwAlye5NckOSb6fZCDJh5J8ati6H0jy+W76xCQ3d+v8ffcIXWlMGNyarI4CvlNVvwBWJzkwyZFd/eCqeiPwqaq6ClgCnFBV+1fV2mHbuAp4z7D5Y4FvJnlDN/2WqtofeAE4YQsckyaJZn4sWBpjxwGf7aav6OZfBny1qp4BqKqNPk+6qlYluS/JIcA9wOuB/wOcCrwJuCUJwA7Ayn4chCYng1uTTpJXAO8A/ihJAdsw9Djbq3npj7X9JvA+4C7gmqqqDKX14qo6cwzbltbzUokmo6OBS6tq76qaU1WzgfsZ+sWWP0uyI0CS3bvxTwK7jLKtbzF0eeU4hkIc4Abg6CR7rNtOkr37cyiajAxuTUbHAddsULuaoTtHrgOWJLkV+E/dsq8BX1r34eTwlapqDbAU2Luqbu5qS4GzgH9KchswCOzZp2PRJOTTASWpMZ5xS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhrz/wHi7yc4Q1JV1wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#store the activity labels of all compounds\n",
    "y = p53.Active.values\n",
    "\n",
    "#return the count for each unique value (0 and 1)\n",
    "unique, counts = np.unique(y, return_counts=True)\n",
    "dict(zip(unique, counts))\n",
    "\n",
    "#plot the label counts \n",
    "count1 = sns.catplot(x=\"Active\", kind=\"count\", palette=\"ch:.25\", data=p53)\n",
    "\n",
    "#I added the count of class labels as the title here\n",
    "count1.fig.suptitle('0: 6406, 1: 420')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Methods of class balancing\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Morgan distribution: [('0', 6406), ('1', 6414)] Daylight-like distribution: [('0', 6406), ('1', 6385)] Atom-pair distribution: [('0', 6406), ('1', 6385)] Torsion distribution: [('0', 6406), ('1', 6385)]\n"
     ]
    }
   ],
   "source": [
    "x_morg_rsmp, y_morg_rsmp = ADASYN().fit_resample(x_morg, y)\n",
    "morg_sample_count = sorted(Counter(y_morg_rsmp).items())\n",
    "\n",
    "x_rd_rsmp, y_rd_rsmp = ADASYN().fit_resample(x_rd, y)\n",
    "rd_sample_count = sorted(Counter(y_rd_rsmp).items())\n",
    "\n",
    "x_AP_rsmp, y_AP_rsmp = ADASYN().fit_resample(x_AP, y)\n",
    "AP_sample_count = sorted(Counter(y_rd_rsmp).items())\n",
    "\n",
    "x_torsion_rsmp, y_torsion_rsmp = ADASYN().fit_resample(x_torsion, y)\n",
    "torsion_sample_count = sorted(Counter(y_rd_rsmp).items())\n",
    "\n",
    "print(\"Morgan distribution:\", morg_sample_count, \n",
    "      \"Daylight-like distribution:\", rd_sample_count, \n",
    "      \"Atom-pair distribution:\", AP_sample_count, \n",
    "      \"Torsion distribution:\", torsion_sample_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**We can plot one of the re-sampled distributions to visually represent class balance:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAFgCAYAAACbqJP/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE6RJREFUeJzt3XGsnfV93/H3BxyatE2KSQyjthPYYqWh00LYFXhDmrYwGcO2mmXQgZLFoZ48TaxqpU0dmSaxQSOl0bY0iVI6KzgxUVaCSBleFUEtJ1nVdQSchZJgJ7NHOrgyxZdeh6ZlTeXsuz/u7yYHuPf6wO5z7/35vF/S1TnP7/zOub+LrDePnvOc56SqkCT146zVXoAk6ZUx3JLUGcMtSZ0x3JLUGcMtSZ0x3JLUGcMtSZ0x3JLUGcMtSZ1Zt9oLGML27dvrwQcfXO1lSNIrlXEmnZF73M8999xqL0GSBnNGhluSzmSGW5I6Y7glqTOGW5I6Y7glqTOGW5I6Y7glqTOGW5I6Y7glqTOGW5I6Y7glqTOGW5I6c0ZeHVBaq6beMrXaS9DADv3vQ4P/DsP9Ek8deXi1l6AV8Oa3b13tJUivmodKJKkzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOjNouJOcm+S+JN9MciTJX0tyXpIDSY622/VtbpJ8LMmxJI8nuWzkdXa2+UeT7BxyzZK01g29x/1R4MGq+ingHcAR4FbgYFVtAQ62bYBrgC3tZzdwJ0CS84DbgCuAy4Hb5mMvSZNosHAneQPwN4C7AKrqz6vqO8AOYF+btg+4rt3fAdxdcx4Gzk1yIXA1cKCqZqvqJHAA2D7UuiVprRtyj/svAjPAp5J8Lcknk/wYcEFVPQPQbs9v8zcCT488f7qNLTb+Ikl2JzmU5NDMzMzy/zWStEYMGe51wGXAnVX1TuBP+eFhkYVkgbFaYvzFA1V7qmqqqqY2bNjwatYrSV0YMtzTwHRVfaVt38dcyJ9th0BotydG5m8eef4m4PgS45I0kQYLd1X9IfB0kre1oauAw8B+YP7MkJ3AA+3+fuB97eySrcDz7VDKQ8C2JOvbm5Lb2pgkTaShv+X954HPJjkHeBK4mbn/WdybZBfwFHBDm/sF4FrgGPBCm0tVzSa5A3i0zbu9qmYHXrckrVmDhruqHgOmFnjoqgXmFnDLIq+zF9i7vKuTpD75yUlJ6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTODBruJH+Q5OtJHktyqI2dl+RAkqPtdn0bT5KPJTmW5PEkl428zs42/2iSnUOuWZLWupXY4/5bVXVpVU217VuBg1W1BTjYtgGuAba0n93AnTAXeuA24ArgcuC2+dhL0iRajUMlO4B97f4+4LqR8btrzsPAuUkuBK4GDlTVbFWdBA4A21d60ZK0Vgwd7gJ+O8lXk+xuYxdU1TMA7fb8Nr4ReHrkudNtbLHxF0myO8mhJIdmZmaW+c+QpLVj3cCvf2VVHU9yPnAgyTeXmJsFxmqJ8RcPVO0B9gBMTU297HFJOlMMusddVcfb7QngfuaOUT/bDoHQbk+06dPA5pGnbwKOLzEuSRNpsHAn+bEkr5+/D2wDvgHsB+bPDNkJPNDu7wfe184u2Qo83w6lPARsS7K+vSm5rY1J0kQa8lDJBcD9SeZ/z3+qqgeTPArcm2QX8BRwQ5v/BeBa4BjwAnAzQFXNJrkDeLTNu72qZgdctyStaYOFu6qeBN6xwPgfAVctMF7ALYu81l5g73KvUZJ65CcnJakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOmO4JakzhluSOjN4uJOcneRrSX6rbV+c5CtJjib5XJJz2viPtO1j7fGLRl7jA238W0muHnrNkrSWrcQe9y8AR0a2fwX4SFVtAU4Cu9r4LuBkVb0V+EibR5JLgBuBnwa2A7+W5OwVWLckrUmDhjvJJuDvAJ9s2wHeBdzXpuwDrmv3d7Rt2uNXtfk7gHuq6ntV9W3gGHD5kOuWpLVs6D3uXwV+Cfi/bfuNwHeq6lTbngY2tvsbgacB2uPPt/k/GF/gOT+QZHeSQ0kOzczMLPffIUlrxmDhTvJ3gRNV9dXR4QWm1mkeW+o5Pxyo2lNVU1U1tWHDhle8XknqxboBX/tK4GeSXAu8FngDc3vg5yZZ1/aqNwHH2/xpYDMwnWQd8BPA7Mj4vNHnSNLEGWyPu6o+UFWbquoi5t5c/GJVvQf4EnB9m7YTeKDd39+2aY9/saqqjd/Yzjq5GNgCPDLUuiVprRtyj3sx/xK4J8kvA18D7mrjdwGfSXKMuT3tGwGq6okk9wKHgVPALVX1/ZVftiStDSsS7qr6MvDldv9JFjgrpKr+DLhhked/EPjgcCuUpH74yUlJ6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOGG5J6ozhlqTOjBXuJAfHGZMkDW/JqwMmeS3wo8Cbkqznh99G8wbgJwdemyRpAae7rOs/AX6RuUh/lR+G+4+BTwy4LknSIpYMd1V9FPhokp+vqo+v0JokSUsY64sUqurjSf46cNHoc6rq7oHWJUlaxFjhTvIZ4C8BjwHzXxtWgOGWpBU27leXTQGXtC/vlSStonHP4/4G8BeGXIgkaTzj7nG/CTic5BHge/ODVfUzg6xKkrSoccP9b4ZchCRpfOOeVfJfh16IJGk8455V8l3mziIBOAd4DfCnVfWGoRYmSVrYuHvcrx/dTnIdcPkgK5IkLelVXR2wqv4z8K5lXoskaQzjHip598jmWcyd1+053ZK0CsY9q+Tvjdw/BfwBsGPZVyNJOq1xj3HfPPRCJEnjGfeLFDYluT/JiSTPJvl8kk1DL06S9HLjvjn5KWA/c9fl3gj8lzYmSVph44Z7Q1V9qqpOtZ9PAxsGXJckaRHjhvu5JO9Ncnb7eS/wR0MuTJK0sHHD/XPAzwJ/CDwDXA/4hqUkrYJxTwe8A9hZVScBkpwH/Dvmgi5JWkHj7nH/lfloA1TVLPDOYZYkSVrKuOE+K8n6+Y22x73k3nqS1yZ5JMnvJ3kiyb9t4xcn+UqSo0k+l+ScNv4jbftYe/yikdf6QBv/VpKrX+kfKUlnknHD/e+B30tyR5Lbgd8DPnya53wPeFdVvQO4FNieZCvwK8BHqmoLcBLY1ebvAk5W1VuBj7R5JLkEuBH4aWA78GtJzh73D5SkM81Y4W7f5v4PgGeBGeDdVfWZ0zynqupP2uZr2k8xd3Gq+9r4PuC6dn9H26Y9flWStPF7qup7VfVt4BhemVDSBBv3zUmq6jBw+JW8eNsz/irwVuATwP8CvlNVp9qUaeY+0EO7fbr9rlNJngfe2MYfHnnZ0eeM/q7dwG6AN7/5za9kmZLUlVd1WddxVdX3q+pSYBNze8lvX2hau80ijy02/tLftaeqpqpqasMGPxsk6cw1aLjnVdV3gC8DW4Fzk8zv6W8Cjrf708BmgPb4TwCzo+MLPEeSJs5g4U6yIcm57f7rgL8NHAG+xNwHeAB2Ag+0+/vbNu3xL1ZVtfEb21knFwNbgEeGWrckrXVjH+N+FS4E9rXj3GcB91bVbyU5DNyT5JeBrwF3tfl3AZ9Jcoy5Pe0bAarqiST3Mnd8/RRwS1V9f8B1S9KaNli4q+pxFviQTlU9yQJnhVTVnwE3LPJaHwQ+uNxrlKQercgxbknS8jHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktSZwcKdZHOSLyU5kuSJJL/Qxs9LciDJ0Xa7vo0nyceSHEvyeJLLRl5rZ5t/NMnOodYsST0Yco/7FPDPq+rtwFbgliSXALcCB6tqC3CwbQNcA2xpP7uBO2Eu9MBtwBXA5cBt87GXpEk0WLir6pmq+h/t/neBI8BGYAewr03bB1zX7u8A7q45DwPnJrkQuBo4UFWzVXUSOABsH2rdkrTWrcgx7iQXAe8EvgJcUFXPwFzcgfPbtI3A0yNPm25ji41L0kQaPNxJfhz4PPCLVfXHS01dYKyWGH/p79md5FCSQzMzM69usZLUgUHDneQ1zEX7s1X1m2342XYIhHZ7oo1PA5tHnr4JOL7E+ItU1Z6qmqqqqQ0bNizvHyJJa8iQZ5UEuAs4UlX/YeSh/cD8mSE7gQdGxt/Xzi7ZCjzfDqU8BGxLsr69KbmtjUnSRFo34GtfCfwj4OtJHmtj/wr4EHBvkl3AU8AN7bEvANcCx4AXgJsBqmo2yR3Ao23e7VU1O+C6JWlNGyzcVfW7LHx8GuCqBeYXcMsir7UX2Lt8q5OkfvnJSUnqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4YbknqjOGWpM4MFu4ke5OcSPKNkbHzkhxIcrTdrm/jSfKxJMeSPJ7kspHn7GzzjybZOdR6JakXQ+5xfxrY/pKxW4GDVbUFONi2Aa4BtrSf3cCdMBd64DbgCuBy4Lb52EvSpBos3FX1O8DsS4Z3APva/X3AdSPjd9ech4Fzk1wIXA0cqKrZqjoJHODl/zOQpImy0se4L6iqZwDa7fltfCPw9Mi86Ta22LgkTay18uZkFhirJcZf/gLJ7iSHkhyamZlZ1sVJ0lqy0uF+th0Cod2eaOPTwOaReZuA40uMv0xV7amqqaqa2rBhw7IvXJLWipUO935g/syQncADI+Pva2eXbAWeb4dSHgK2JVnf3pTc1sYkaWKtG+qFk/wG8DeBNyWZZu7skA8B9ybZBTwF3NCmfwG4FjgGvADcDFBVs0nuAB5t826vqpe+4SlJE2WwcFfVTYs8dNUCcwu4ZZHX2QvsXcalSVLX1sqbk5KkMRluSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakzhhuSeqM4ZakznQT7iTbk3wrybEkt672eiRptXQR7iRnA58ArgEuAW5KcsnqrkqSVkcX4QYuB45V1ZNV9efAPcCOVV6TJK2Kdau9gDFtBJ4e2Z4GrhidkGQ3sLtt/kmSb63Q2s4EbwKeW+1F6Iw0cf+2kvz/PP3Bqtp+ukm9hHuh/xL1oo2qPcCelVnOmSXJoaqaWu116Mzjv61h9HKoZBrYPLK9CTi+SmuRpFXVS7gfBbYkuTjJOcCNwP5VXpMkrYouDpVU1akk/wx4CDgb2FtVT6zyss4kHmLSUPy3NYBU1elnSZLWjF4OlUiSGsMtSZ0x3BPOSwloCEn2JjmR5BurvZYzkeGeYF5KQAP6NHDaD5Lo1THck81LCWgQVfU7wOxqr+NMZbgn20KXEti4SmuRNCbDPdlOeykBSWuP4Z5sXkpA6pDhnmxeSkDqkOGeYFV1Cpi/lMAR4F4vJaDlkOQ3gP8OvC3JdJJdq72mM4kfeZekzrjHLUmdMdyS1BnDLUmdMdyS1BnDLUmdMdyaWEn+fpJK8lOnmff+JD85sv1JL8al1WS4NcluAn6XuQ8eLeX9wA/CXVX/uKoOD7guaUmGWxMpyY8DVwK7GAl3kl9K8vUkv5/kQ0muB6aAzyZ5LMnrknw5yVSSf5rkwyPPfX+Sj7f7703ySHvOf2yX0JWWheHWpLoOeLCq/icwm+SyJNe08Suq6h3Ah6vqPuAQ8J6qurSq/s/Ia9wHvHtk+x8Cn0vy9nb/yqq6FPg+8J4V+Js0Ibr4lndpADcBv9ru39O2zwI+VVUvAFTVkteTrqqZJE8m2QocBd4G/DfgFuCvAo8mAXgdcGKIP0KTyXBr4iR5I/Au4C8nKeBs5i5n+3le+WVtPwf8LPBN4P6qqszVel9VfWAZly39gIdKNImuB+6uqrdU1UVVtRn4NnPf2PJzSX4UIMl5bf53gdcv8lq/ydzhlZuYizjAQeD6JOfPv06Stwzzp2gSGW5NopuA+18y9nnmzhzZDxxK8hjwL9pjnwZ+ff7NydEnVdVJ4DDwlqp6pI0dBv418NtJHgcOABcO9LdoAnl1QEnqjHvcktQZwy1JnTHcktQZwy1JnTHcktQZwy1JnTHcktSZ/wfEJtFwrmegugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_morg_rsmp_df = pd.DataFrame(y_morg_rsmp, columns=['Active'])\n",
    "sns.catplot(x=\"Active\", kind=\"count\", palette=\"ch:.25\", data=y_morg_rsmp_df);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating training, testing, and validation test sets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Partitioning the data into an 85/15 train/test split\n",
    "x_morg_train, x_morg_test, y_morg_train, y_morg_test = train_test_split(x_morg_rsmp, y_morg_rsmp, test_size=0.15, random_state=1)\n",
    "#The training data is further split into an 80/15 train/validtion split \n",
    "x_morg_train, x_morg_val, y_morg_train, y_morg_val = train_test_split(x_morg_train, y_morg_train, test_size=0.15, random_state=1)\n",
    "\n",
    "x_rd_train, x_rd_test, y_rd_train, y_rd_test = train_test_split(x_rd_rsmp, y_rd_rsmp, test_size=0.15, random_state=1)\n",
    "x_rd_train, x_rd_val, y_rd_train, y_rd_val = train_test_split(x_rd_train, y_rd_train, test_size=0.15, random_state=1)\n",
    "\n",
    "x_AP_train, x_AP_test, y_AP_train, y_AP_test = train_test_split(x_AP_rsmp, y_AP_rsmp, test_size=0.15, random_state=1)\n",
    "x_AP_train, x_AP_val, y_AP_train, y_AP_val = train_test_split(x_AP_train, y_AP_train, test_size=0.15, random_state=1)\n",
    "\n",
    "x_torsion_train, x_torsion_test, y_torsion_train, y_torsion_test = train_test_split(x_torsion_rsmp, y_torsion_rsmp, test_size=0.15, random_state=1)\n",
    "x_torsion_train, x_torsion_val, y_torsion_train, y_torsion_val = train_test_split(x_torsion_train, y_torsion_train, test_size=0.15, random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross validation and model fitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Model with Cross Validation is: 95.69204220812904 (+/- 0.00)\n"
     ]
    }
   ],
   "source": [
    "#set our linear regression function\n",
    "lr = LogisticRegression(solver = 'lbfgs', max_iter = 1000)\n",
    "#fit the data \n",
    "#perform 10 fold crossvalidation to estimate accuracy of model\n",
    "accuracy_morg = cross_val_score(lr, x_morg_train, y_morg_train, scoring='accuracy', cv = 10)\n",
    "\n",
    "#get the mean of each fold \n",
    "print(\"Accuracy of Model with Cross Validation is:\",accuracy_morg.mean() * 100, \"(+/- %0.2f)\" % accuracy_morg.std())\n",
    "\n",
    "#Create our model using 10 fold cross valiation, lbfgs, and 1000 iterations - and fit the data to out training set\n",
    "lr_morg_ = LogisticRegressionCV(cv=10, solver = 'lbfgs', max_iter = 1000).fit(x_morg_train, y_morg_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the rest of the fingerprints we can simply write a function to model the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function to train the logistic regression classifier on given x and y training sets\n",
    "def lrCV_model(x_train, y_train):\n",
    "    lr = LogisticRegression(solver = 'lbfgs', max_iter = 650)\n",
    "    model_accuracy = cross_val_score(lr, x_train, y_train, scoring='accuracy', cv = 5)\n",
    "    accuracy_printout = \"Accuracy of Model with Cross Validation is:\",model_accuracy.mean() * 100, \"(+/- %0.2f)\" % model_accuracy.std()\n",
    "    lr_fit = LogisticRegressionCV(cv=5, solver = 'lbfgs', max_iter = 650, multi_class = \"ovr\", n_jobs = -1).fit(x_train, y_train)\n",
    "    \n",
    "    return lr_fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train logistic regression models for all fingerprints\n",
    "lr_morg = lrCV_model(x_morg_train, y_morg_train)\n",
    "lr_rd = lrCV_model(x_rd_train, y_rd_train)\n",
    "lr_AP = lrCV_model(x_AP_train, y_AP_train)\n",
    "lr_torsion = lrCV_model(x_torsion_train, y_torsion_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Metrics and prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test set predictions and confusion matrix\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[909  66]\n",
      " [  0 948]]\n"
     ]
    }
   ],
   "source": [
    "#predicting using the test set and creating a confusion matrix\n",
    "predictions_morg_test = lr_morg.predict(x_morg_test)\n",
    "\n",
    "#confustion matrix for test set predictions\n",
    "cm_morg_test = metrics.confusion_matrix(y_morg_test, predictions_morg_test)\n",
    "\n",
    "print(cm_morg_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Validation set predictions and confusion matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[772  74]\n",
      " [  0 789]]\n"
     ]
    }
   ],
   "source": [
    "#predicting using the validation set and creating a confusion matrix\n",
    "predictions_morg_val = lr_morg.predict(x_morg_val)\n",
    "\n",
    "#confustion matrix for validation set predictions\n",
    "cm_morg_val = metrics.confusion_matrix(y_morg_val, predictions_morg_val)\n",
    "\n",
    "print(cm_morg_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Model accuracy on test set**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of logistic regression classifier on train set: 0.9990282876268625\n"
     ]
    }
   ],
   "source": [
    "score_morg_train = lr_morg.score(x_morg_train, y_morg_train)\n",
    "print('Accuracy of logistic regression classifier on train set:', score_morg_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Model accuracy on test set**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of logistic regression classifier on test set: 0.9656786271450858\n"
     ]
    }
   ],
   "source": [
    "score_morg_test = lr_morg.score(x_morg_test, y_morg_test)\n",
    "print('Accuracy of logistic regression classifier on test set:', score_morg_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Model accuracy on validation set**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of logistic regression classifier on test set: 0.9547400611620795\n"
     ]
    }
   ],
   "source": [
    "score_morg_val = lr_morg.score(x_morg_val, y_morg_val)\n",
    "print('Accuracy of logistic regression classifier on test set:', score_morg_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Classification report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.91      0.95       846\n",
      "           1       0.91      1.00      0.96       789\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      1635\n",
      "   macro avg       0.96      0.96      0.95      1635\n",
      "weighted avg       0.96      0.95      0.95      1635\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_morg_val, predictions_morg_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgoAAAIBCAYAAAAoBpeWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xe8HFX9//HXJwmBQEhCSSNBQglNpFcLHaQTmuJPlKZRBBUR6QpqQEDEhoiRrnzphCKd0EQQAenSkZIeSCWEEnJ+f8xc2Gzu3nszZrObzOuZxzxyZ+bMzJndvXfPvs+Z2UgpIUmS1JpOja6AJElqXjYUJElSTTYUJElSTTYUJElSTTYUJElSTTYUJElSTTYUJElSTTYUJElSTTYUJElSTTYUJElSTV0aXQFJkhZW782i7t+DsEQXot7HaIuJgiRJqslEQZKkgsrwvYomCpIkqSYTBUmSCkr1H6IAjR2iYKIgSZJqM1GQJKkoxyhIkqQyM1GQJKmgEgQKJgqSJKk2EwVJkgryPgqSJKnUTBQkSSqoDPdRsKEgSVJRdj1IkqQyM1GQJKmgEgQKJgqSJKk2EwVJkgry8khJklRqJgqSJBW0YC6PbCwTBUmSVJOJgiRJBTlGQZIklZoNBUmSVJMNBUmSVJMNhQaLzH8jIkXEao2uT7OJiOUj4pyIeDUi3ouIMRFxe0QMaXTdioqIz0XEwxExM3/uv9fB7T4fEQ9VPA6nRsRc44wiYqWIuDwiJkXEuxHxZETsVLF+6/z1Vj2dXmBfF9fYV4qIr1SU6xERv4mI1/L9PBcRR0bEXN92ExFDI+KZ/DzHR8SVVfv5aUT8KyKmRsS4iBgREatX7eOUNup1fEW5HfLzey1fd0o7z0GniHgsL7tb1bpaj8WaVeV6RsRFETE5P4fLImK5qjJdI+InEfFy/jp5OT/vxSvKDKpxvCvmdV8qLqX6T43mYMbG2wIYlP+8PzCscVVpLhGxGHAPsCRwKvAKMBDYEdgOuL5xtSsmbwzeDvwNOB7YFDg7It5NKZ3fxnYrA3fm2+4FrAb8AlgKOLKi3IrAQ8CTwMHADGB9oFsru/0q8GrF/OiqY3ZkXz8Hzqva72HA/wPuqlh2MbAlcALwMrANcDbZ1+L9uuKYw4AjyH4PHgH6AltV7OdTwDeBC4ATyV4bxwMPR8S6KaU383LnA7dV1WsIcCxwa8WynYB1gZFkv3/t+QYwoI31z5M9VpVeq5q/Elgj39ds4Ayy1/IXKsqcDnwbOAl4HNiQ7DHpBXy/an9HA/+omH+rav287EuaW0rJqYET8HvgHeCfwLONrk9V3ZZo8PF3ILuV+iatrIsFcPxuddjnn4AXgS4Vy84F3mzrnPLtXq3a7nvAh0D/imVXAH8HOrWxr63zx3Wddura7r5qbPcscGvF/JLAR8B3q8pdBzxcMf/pvNwObex7qernBVg2/x06uZ163Qw8V7WsU8XPbwGntLH9MsBE4ND88dutav3FwKPt1GGLfNstK5Ztmi/bvmLZOOBXVdueDYyvmB/UWj1aOWa7+3IqPr31zoep3lOjz9GuhwaKiM7AfsCNwIXA2hGxbivlWuLft/LY9qmI+H8V67tFxJkR8XpEvB9ZnP2LivUpIo6o2ucpEfFWxfxBeblNI+LeiJgJ/Chfd3pEPB0R70TEqDwq7ddKPb+Zl2uJjK/JY9ZdI2J2/qm4svzK+fI9ajxEvfL/x1WvSPlfu4p9rRsRN0XElLye/4qIHaqOdX1ETIuI6XnZ1ar2kSLiqMgi8onA0xXr9oyIR/NzG5c/3ovVqHdbdgauSynNqlh2BVlSsk4b260P3Fu13R1kqeCOeR17AnsD56aUZheo28eK7it//a4NXF6xuAtZN+fUquJTyBKFFgcCr6SU7qy1/5TSjJTSzKplk4DXgT5t1GtZsoZnZb2Yx8fp52Sf3EfOwzbVdiZ7g76/og7/Av6br2uxGO0/Xh01P/elErKh0FjbkkWrVwDXkH06/EplgYjoQxb/bkIWMe5OFruumK8P4AayuPcPwC7AycDyBet0OVksvkv+P2R/gE8DdiWLuVcB7s4bOi31PInsU+99ZBHvYWR/nLqTRcBjyN4IKh1E9gntlhp1eYIsmr0wsv75VrvK8j7gfwD9ySLWvYARfPIYLU72x30tstj6IGBl4L78DaTSj/L9fI3sEzsR8SWyT7//AvYAfgoMJYv+W+rQ0l98UI1zISKWyuv0fNWq5/L/16S2JYAPqpa9n/+/Vv7/hmRvCiki/hERH+YNu+Pz10m1uyPio8j650+qfD4L7KvF/sB7VHQLpZSmAVcBx0TE+hGxdGT9+18ie8222Ax4uqURmzd674qItWhDRPQm64r5TxvF9s3P54o2yrR1jHXJuhSObqfo2nlj9P2IeCAitqpavyZzP/+QvQYqn//zgW9FNp6le0R8gex36pxWtr0ofx7HRsTZEVHdzTQv+9I8SiUYo9Dw2KbME1mKMBnoms/fTPbJIirK/IKsb7h/jX18kSx+3KON4yTgiKplpwBvVcwflJf7fjt17kzWR/txfEr2yf9d4Ow2thtWeW5kn2ZeA85q53hHkb1BJmAmWaNjv6oylwOjqNFVQNZ4mAWsUrFsYL7f46sep8ertg2yT6sXVS0/JK/Pcvn8Svkxvt7GubQ8bkOqlnfJlw9tY9trgceqln053254Pv+VfH4qWb/0NsDPyOL871Rst0H+utoF2B74TV7mtxVlOrSvVur5CnBtK8sXJ2sMp3yaDRxbVeYFYDrZG/5eZI3iJ/PHv2Y3GHAp8HbLc1GjzN3Vj18rZWp2PZA1gM/Mfx5E610P3yd7A96KrGHyUP4a27SizJ3A9a3s/6/Ag1Wvu99VPF4J+EPVNv3J3uz3IOtOOiV/Td7Qymu4zX05FZ8mTv8w1Xtq9Dk2/EEu65T/4ZwMXFix7ID8l3iLimUPt/aHt2L9GcDb7RxrXhoKq7Wy/c7Ag/mbRuUfm29UrE/AZ9qowyr5m8M2+fy2+Taf7sBjNQD4Dtmn0sn5dr+oWD+eqj7Yqu0vBP7VyvJ7gJurHqdhVWXWyJfvTPaG3jK1vFlsNQ/PeUtDYc+q5S0NhW+2se32eZkfk6VFm5M1vGYBf8zLfDUvc0Ur5/9mO3U7gyzRWr7ovsgSgQTs28q6lnEYB/HJoMaZwKEVZV7KXyNrVb1uZlWWq9rvYfk2e7Vxbv3JGjhHt/MYtNpQIEtJxgE98vmW5769sQHd8ufo+opldwIjWil7GfCPivljgElkAzu3BL5L1l3ws3aOeVhet/X/1305dWyaMP3DVO+p0edo10Pj7Ez2SfyWiOgVEb2Ae8ni5Mruh+WAsW3sp73182p85UxEbEI2hmIUWRy/BdmbFGRxeEsdaKseKaVXyc6vZUT4wWRv3s+2V6GU0uiU0rkppS+RJQG3AT+KTy4pa+8x6F99XrnxZAPhqpdVaunCuYXsjbRl+m++fMX26l9hSv5/r6rly1Stn0tK6S6yUesnknXX3E/WBTWpos6T8v/vqdr8bmBgRPRoo27XkDVYWsbIFNnX/mSJwM2VC/PY/jDgkJTSxSml+1NKp5ElGWdFRMvfoUlk/fctXTEtr5vXyMY9zCEf2/J7smRiRBvn9iWyT9VXtlGmVfk4lF+SNaQ65b+nLee+VEQsXWvblI2luIWsG6fFZOZ+/smXTcmPuTxZAndsSumc/PH6PdkVG8fn3ZG1XJP/v+F82JcEOEahkVoaA1eT/fGYTPaJa3HgSxX9xW+TvdHV0t56yBofXauWVb9BtkhV83uRvTF9OaV0Y0rpn8w9uPDt/P/26nE+sE9EDCAbKHdRO+XnrlxKM8g+nXYm65duOX5bxx5L6wPd+vLJG+LHh6iab1k/lGycSPV0Kx2U1/1N5h6L0DLfWt915fankjVc1s3rfh7Qm+yKGfhkrEO1ljEFHRm413L+87Sv/M1+P7JPzzOrtmk5vyeqlj9O9gbZ0uBr65jVx/ss2XiD81JKv6yxXYv9gQfSJ5dOzoulyBqnZ/PJ7+mT+boryM6hPZWvqedpfSxK5diFVcjGU7T2eHUh6+Zq71gt//8v+1JHpAUwNZgNhQaIiO7AbmR969tUTUeRvQlskxcfCXwxIvrW2N1IYNmouvlLlVF8MuCt5Y/6th2sbjcgy74+8dWqMg+RxcgHtrOv68j6bK8ge+21ObAsIpatMYBxcP7/hPz/kWSNqyVaKQtZ981GlVdd5I2VzwIPtFPnF8juLzAopfRoK9Pb7Wxf7VZgr6qBg18ma0A8097GKaV3UkpPp5QmA4eT9d/fla97jezSxO2qNtuO7GqCd9rY9T5kEf9TBfe1JVnXyuXM7fX8/w2rlm9ENv6m5eqbvwH9IuLj9CAiViV7M3uyYtmn87K3kQ84rSUiBpElYK3VqyPeYe7f0ZZG/gnM/btQeexuZMnhYxWLbyU7x89XlNuY7A29pdHZ1uMFc9+XodK++f8tx/xf9qUOSAvgX8M1uu+jjBOf9P9u1sq6xcj+cF6Qz/cme6N/keyNeFuyN4hj8vVB9gdzGtmI7G3z/f+pYp+/JBts+B2yG8xcA7xB62MUulfVZ5d8+W/I3iR+TPbmOce4B7Kb3swmGzS1E7AnMBwYULW/c/Jt/68Dj9O+ZP3WJ5Bd2tZy/BnATRXl1sjP/19kb7rbk129cEi+fnGyexA8TxZD70N26eNoYNmK/cw1liNf/mWyBs7v+WQA4FCyWHnJvMxKtDOYMS+3Gtmbz/+RvekcQ9aV8Y2qcrOAn1Rt95P8sd2NLE34gKp7DpAlQLPz53wHshtVfQR8taLMH8kGJu5ONhj2t3mZ6mvt291XRdnzyF63XVpZ15ns5kmjye5BsG1+Lu+TDxCsKPdY/jztmx//abLBjYvlZfqQNareIBvAt3nFtHYrxz4uf3x713g+VsqPtW/+Groq/3nnNp7DQVSNUQB6kt1z4ltkr9MvkyU97wMbV21/W/563JvsCqEXgL9XlRlB1hXx/fx18gOyMUJXVZQ5BfhVvp/t8+d0JlVjmjqyL6fi0/hpH6R6T40+x4Y/yGWcyD4NvdjG+nPJIs7F8/mVyPpXJ5O94T8J7F9RvhtwFlmD4n2y/vNTK9Z3By4hi9HHkfV1n0IHGgr5umPyP84zyD69Dqb1AZLfyv+ov58f5yryAWAVZVoG5W3fgcdpxfy8nsj/0E0ne+M4nvwNuqLsumRv3NPz6WFgu4r1q5Bdsjed7I36b8Dgqn202lDI1+2cvxHMIHtDeYKs77dLvn5Qvv1BHTivz5M1at4j+0T3vVbKJCoG1pHdkfD+/A/8DLLxHl+osf8DyGL8D8jugvjtqvXfI0sOpufP1bNkl73OdWOl9vaVl+lC1j11Xhvn3I+s6+n1/DX8XP48dq0q15fs0//UvH4jgE9VrN+a2gHtva0c9wngtjbqdVCNfb3WxjYtz3VlQ2EJssTszfwxnUrWINi8le17kXW7TclfS/9HPoi0okwPstf+K2Rv/i8DZwJLV5TZH3g0P1bL8/Mz8r8b87Ivp+LTuKkfpHpPjT7HlkvVpAUiIs4k+7S1cvofbwokSY02ftqHdX8T7dtjsYbeHMvvetACERFrkI1cPwz4qY0ESYuCMnzUtqGgBeVPZNfZ30g2jkGStBCwoaAFIqW0daPrIEnzXQkiBS+PlCRJNZkoSJJUUCpBpNDMDYVF/9GXJNWTX6U9HzRzQ4EpMz9qdBWkptSrW3Zjx27bndbgmkjNaebIExbIccpwhwHHKEiSpJqaOlGQJKmZlSBQMFGQJEm1mShIklSQYxQkSVKpmShIklTYoh8pmChIkqSaTBQkSSrIMQqSJKnUTBQkSSqoBIGCDQVJkoqy60GSJJWaiYIkSQWV4WumTRQkSVJNJgqSJBW16AcKJgqSJKk2EwVJkgoqQaBgoiBJkmozUZAkqSDvoyBJkkrNREGSpIK8j4IkSSo1EwVJkopa9AMFEwVJklSbiYIkSQWVIFAwUZAkSbWZKEiSVJD3UZAkSU0tIn4QEc9GxDMRcXlELBERK0fEwxHxUkRcGRFd87KL5/Mv5+sHtbd/GwqSJBWUFsC/tkTEAOB7wMYppXWAzsD+wBnAr1NKg4HJwKH5JocCk1NKqwG/zsu1yYaCJEkLty5At4joAiwJjAW2Ba7J118CDMl/3jOfJ1+/XUREWzu3oSBJUlFpAUxtHT6l0cBZwBtkDYSpwGPAlJTSrLzYKGBA/vMA4M1821l5+eXaOoYNBUmSmlhEDI2IRyumoRXrliFLCVYGVgCWAnZuZTctTY7W0oM2myNe9SBJUkEL4qKHlNJwYHiN1dsD/00pTQSIiOuAzwK9IqJLnhoMBMbk5UcBKwKj8q6KnsCkto5voiBJUkEp1X9qxxvA5hGxZD7WYDvgP8A9wL55mQOBG/Kfb8znydffnVLbR7GhIEnSQiql9DDZoMR/A0+Tva8PB44FjoqIl8nGIFyQb3IBsFy+/CjguPaOYdeDJEkFNcPXTKeUTgZOrlr8KrBpK2XfA/abl/2bKEiSpJpMFCRJKqrxgULdmShIkqSaTBQkSSqoBIGCiYIkSarNREGSpIL8mmlJklRqJgqSJBXUDPdRqDcTBUmSVJOJgiRJRS36gYKJgiRJqs1EQZKkgkoQKJgoSJKk2kwUJEkqyPsoSJKkUjNRkCSpIO+jIEmSSs1EQZKkohb9QMGGgiRJRZWgnWDXgyRJqs1EQZKkgrw8UpIklZqJgiRJBXl5pCRJKjUTBUmSilr0AwUTBUmSVJuJgiRJBZUgUDBRkCRJtZkoSJJUkPdRkCRJpWaiIElSQd5HQZIklZqJgiRJRS36gYKJgiRJqs1EQZKkgkoQKJgoSJKk2kwUJEkqyPsoSJKkUjNRkCSpoDLcR8GGgiRJRS367QS7HiRJUm0mCpIkFVSCQMFEQZIk1WaiIElSQbNLcH2kiYIkSarJREGSpIIW/TzBREGSJLXBREGSpIJKMETBREGSJNVmoiBJUkFluIWziYIkSarJREGSpIJmL/qBgomCJEmqzURBkqSCHKMgSZJKzURBkqSCvI+CJEkqNRMFSZIKKsMYBRsKi7grLvsLN1x3NSkl9tx7P75ywNeZOnUKJx3zQ8aMGc0KKwzg1F+eTY8ePZk2bSrDTj6J0aPepGvXxTnpp8NYdbXBc+1zzOhRnHTsD5k6dSprrrU2p5x6Oost1pUPPviAn550HM8/9yw9e/Zi2Blns8KAAQBcfMFwbrr+Wjp16swPjz2BzT/7+QX9UEjt6rnU4vzx6F1Ze1BvUkp8+6ybefg/ozlsyMZ8e8hGzPpoNrc9/DInDr9nrm132GQVzjp8Bzp3Ci6+5UnOuuIhAFbq15O/nDSEZZbuxhMvjeOQ02/kw1mz6bpYZy44dnc2WL0fk6bN5ICfX88b46cu6FOW2mXXwyLslZdf4obrruaiv17JX68awT/+fi9vvP4al154PhtvtjnX3nQbG2+2OZdeeD4AF58/nNXXWJPLrr6ek4f9grPPPK3V/Z7zm1+x/wEHcu1Nt7F0jx7cOOI6AG4ccS1L9+jBtTfdzv4HHMgffvsrAF595WXuvP1WLr/2Jn577nDOPO3nfPTRRwvmQZDmwVlH7MAdj7zC+gf/iU2Hns/zr7/FluuvxG6fHcwm3zyfjQ79M7+56uG5tuvUKfjN977InsdfyQaHDGe/bddmzZWWB+DUb27L7699hM8ceB6T33mPg3ZeH4CDdl6Pye+8xzpfP4/fX/sIp35zmwV6rpo/Zqf6T41Wt4ZCRKwZEcdGxO8i4rf5z2vV63ia22uvvsI6667HEt260aVLFzbYaBPuu3sk9997N7vuPgSAXXcfwn33jATgv6++wsabbQ7AoJVXYeyYMbz99ltz7DOlxKOPPMy22+841/aV+912+x155F//JKXE/ffezQ5f3JmuXbuywoCBDFzxU/znmacXyGMgddTSS3bl85/5FBff8iQAH86azdQZ7zN09w0564qH+ODDrHE7ccq7c227yZor8Mroybw2dgofzprN1ff8h90+m6VxW22wEtfd9xwAl93xNLt/bnUAdvvs6lx2R/Z7cN19z7H1hoPqfYqqg7QA/jVaXRoKEXEscAUQwL+AR/KfL4+I4+pxTM1tldUG8/hjjzJ1yhTemzmTBx+4n/HjxzLp7bdZvndvAJbv3ZvJkyYBMHj1Nbh35F0APPv0U4wbO4YJ48fPsc+pU6aw9NJL06VL1mvVp29fJk7IykycMJ4+/foB0KVLF7p3X5qpU6YwccIE+ubLW7aZMGHO/UqNtnL/Xrw19V2GH7MbD513COf+cBeWXGIxVhu4LJ/7zIrcf86B3HH2AWy0Rv+5tl1h+aUZNXHax/OjJ05nwPJLs1yPbkx95z0+yj8Wjp44jRWWX/qTbSZk23w0OzFtxvss16PbAjhTad7Ua4zCocCnU0ofVi6MiLOBZ4HT63RcVVh5lVX5+sHf4LvfPpRuSy7J4NXXoHPn2k/51w/5JmefeRoHfGkvVh28OquvsRadO3eeo0xq5VqgiGhzXVvbSM2iS+dOrD+4H0f9/g4eeX4MZx2+A0fvvwVdOndime5LsOURl7DxGv3564/3Yq0Dzp1j29ZezSm1/jpv+X1o7Veg8Z8dNa+8PLK42cAKrSzvn69rVUQMjYhHI+LR4cOH16lq5bLHXvtw6RXX8qcL/0KPHj1Z8VMrsexyy/HWxIkAvDVxIsssuywA3bt35yc/O42/XjWCU4adzpTJk1hhwMA59tdrmWWYPn06s2bNAmDC+PEs37sPAH369mPCuHEAzJo1i3femU6Pnj3p07cv4/PlLdv0zreRmsXoidMZPXEajzw/BoAR9z/P+oP7MXriNK5/4AUAHn1hLLNTYvmeS8657VvTGdi7x8fzA3ovzZi3p/PW1Hfp2X0JOneKfHkPxr79zsfHG9gn26Zzp6DHUoszadrMup+nNK/q1VA4EhgZEbdGxPB8ug0YCXy/1kYppeEppY1TShsPHTq0TlUrl0mT3gZg3Ngx3Hv3Xey48y58YattuPmm6wG4+abr2XLrbQGYPm0aH374AQA3XHcN62+0Md27d59jfxHBRhtvyt133THX9pX7vfuuO9h4k82ICLbcahvuvP1WPvjgA8aMHsWbb7zO2ut8pv4nL82D8ZNnMGridAYPzBrOW28wiOdff4ub/vEiW28wCIDVBi5L1y6deWvqnOMUHn1+DKsNWIaV+vVksS6d2G+btbn5wZcAuP+J19l7q2x41ld3/Ax/e/BFAG5+6CW+umP2e7D3Vmtx3+OvL4jT1HyWUv2nRovWYuH5suOITsCmwACyZG4U8EhKqaPD3dOUmY6M/18NPfgApk6dQpcui3HkD49hk822YOqUKZxwzA8YN3Ys/fr357Rf/pqePXvx9JNPcMpJx9G5c2dWXmVVTjzl5/To0ROAIw//Fiee/HN69+nD6FFvctKxRzNt2hRWX2MtfnramXTt2pX333+fU048lhdfeI4ePXox7IyzGDBwRQAu+vN53HTDCDp37swPfnQcn/38lo18WBZ6vbplXULdtmv9yhQVs+6qfTj3h7vSdbHOvDZ2MkPPvJkZ733An360G+uu2pcPZn3E8eeN5L4nXqf/ct0594e7sNcJVwHwxU1X5ZeHb0/nTp245NYnOfP/HgRgUP9e+eWRS/Dky+M5+Bc38sGHH7H4Yp258Pg9WG+1vkye/h5fG3Y9r42d0sjTX6TMHHkCtN4rNF/d9uzEur+V7/Tp3g3tq61bQ2E+sKEg1WBDQWrbgmoo3PLshLq/ie7y6T4NbSh4HwVJklSTd2aUJKmg5g3l5x8TBUmSVJOJgiRJBTXDnRPrzURBkiTVZKIgSVJBjlGQJEmlZqIgSVJBsx2jIEmSysxEQZKkghyjIEmSSs1EQZKkgkoQKJgoSJKk2kwUJEkqqIm/gXm+MVGQJEk1mShIklTQ7EZXYAGwoSBJUkF2PUiSpFIzUZAkqaBFP08wUZAkSW0wUZAkqSDHKEiSpFIzUZAkqaAyXB5poiBJkmoyUZAkqSDHKEiSpFIzUZAkqaASBAomCpIkLcwioldEXBMRz0fEcxGxRUQsGxF3RsRL+f/L5GUjIn4XES9HxFMRsWF7+7ehIElSQWkBTB3wW+C2lNKawHrAc8BxwMiU0mBgZD4PsDMwOJ+GAn9sb+c2FCRJWkhFRA9gS+ACgJTSBymlKcCewCV5sUuAIfnPewKXpsw/gV4R0b+tYzhGQZKkgmY3fpDCKsBE4KKIWA94DPg+0DelNBYgpTQ2Ivrk5QcAb1ZsPypfNrbWAUwUJElqYhExNCIerZiGVqzuAmwI/DGltAEwg0+6GVrdXSvL2mztmChIklTQgsgTUkrDgeE1Vo8CRqWUHs7nryFrKIyPiP55mtAfmFBRfsWK7QcCY9o6vomCJEkLqZTSOODNiFgjX7Qd8B/gRuDAfNmBwA35zzcCX8+vftgcmNrSRVGLiYIkSQU1yZ0ZvwtcFhFdgVeBg8mCgKsi4lDgDWC/vOwtwC7Ay8C7edk22VCQJGkhllJ6Ati4lVXbtVI2AYfPy/5tKEiSVFAZvj3ShoIkSQU1R89DfTmYUZIk1WSiIElSQU1ww6W6M1GQJEk1mShIklRQCQIFEwVJklSbiYIkSQU5RkGSJJWaiYIkSQXNXvQDBRMFSZJUm4mCJEkFlWCIgomCJEmqzURBkqSCZrPoRwomCpIkqSYTBUmSCnKMgiRJKjUTBUmSCvI+CpIkqdRMFCRJKsjvepAkSaVmoiBJUkElCBRsKEiSVJSDGSVJUqmZKEiSVFAqQd+DiYIkSarJREGSpIIcoyBJkkrNREGSpIJMFCRJUqmZKEiSVFBi0Y8UTBQkSVJNNROFiFi2rQ1TSpPmf3UkSVp4lGGMQltdD48BCYhW1iVglbrUSJIkNY2aDYWU0soLsiKSJC1sSnBjxvbHKETmgIj4cT7/qYjYtP5VkyRJjdaRqx7OBWYD2wI/B6YD1wKb1LFekiQ1vdkliBQ60lDYLKW0YUQ8DpBSmhwRXetcL0mS1AQ60lD4MCI6kw1gJCJ6kyUMkiSVWhmueujIfRREFj2QAAAXIUlEQVR+B4wA+kbEqcADwGl1rZUkSWoK7SYKKaXLIuIxYLt80ZCU0nP1rZYkSc2vBEMUOnwL5yWBlu6HbvWrjiRJaiYduTzyJ8AlwLLA8sBFEXFSvSsmSVKzm51S3adG60ii8BVgg5TSewARcTrwb2BYPSsmSVKza4L38brryGDG14AlKuYXB16pS20kSVJTaetLoX5PNibhfeDZiLgzn9+B7MoHSZJKrQz3Cmir6+HR/P/HyC6PbHFv3WojSZKaSltfCnXJgqyIJEkLm2YYbFhv7Q5mjIjBwC+AtakYq5BS8mumJUlaxHXkqoeLgJOBXwPbAAcDUc9KSZK0MChBoNChqx66pZRGApFSej2ldArZN0lKkqRFXEcShfciohPwUkQcAYwG+tS3WpIkNT+/FCpzJNktnL8HbAR8DTiwnpWSJEnNoSNfCvVI/uM7ZOMTJEkSkEowSKGtGy7dRHaDpVallPaoS40kSVLTaCtROGuB1UKSpIVQGcYotHXDpfsWZEUkSVLz6chVD5IkqRVlSBQ6ctWDJEkqqaZOFHp169zoKkhNbebIExpdBanUvOrBqx4kSSq1pr7qodsGRzS6ClJTmvn4OQC8N6vBFZGa1BILKC+fvWAO01Be9SBJkmrya6YlSSqoDGMUOnLVw0XAH4FZZF8zfSnwl3pWSpIkNQe/ZlqSpIJSqv/UaH7NtCRJBc1uhnfyOvNrpiVJUk1+zbQkSQWVIFDo0FUP99DKjZdSSo5TkCRpEdeRMQpHV/y8BLAP2RUQkiSVWhkuj+xI18NjVYv+ERHejEmSpBLoSNfDshWzncgGNParW40kSVpIlCBQ6FDXw2NkYxSCrMvhv8Ch9ayUJElqDh1pKKyVUnqvckFELF6n+kiStNDwPgqZB1tZ9tD8rogkSWo+NROFiOgHDAC6RcQGZF0PAD3IbsAkSVKpLfp5QttdD18EDgIGAr/ik4bCNOCE+lZLkiQ1g5oNhZTSJcAlEbFPSunaBVgnSZIWCmW4j0JHxihsFBG9WmYiYpmIGFbHOkmSpCbRkYbCzimlKS0zKaXJwC71q5IkSQuH2an+U6N1pKHQufJyyIjoBnh5pCRJJdCR+yj8FRgZEReRDfA8BLi0rrWSJGkhUIYxCh35roczI+IpYHuyKx9+nlK6ve41kyRJDdeRRIGU0m3AbQAR8bmI+ENK6fC61kySpCZXgkChYw2FiFgf+ArwZbLveriunpWSJEnNoa07M64O7E/WQHgbuBKIlNI2C6hukiQ1tbKPUXge+Duwe0rpZYCI+MECqZUkSQuBZrh8sd7aujxyH2AccE9E/DkituOT2zhLkqQSaOsWziOAERGxFDAE+AHQNyL+CIxIKd2xgOooSVJTKkPXQ7s3XEopzUgpXZZS2o3sC6KeAI6re80kSVLDdeTOjB9LKU1KKf0ppbRtvSokSdLCIi2AqdHmqaEgSZLKpUP3UZAkSXOb7RgFSZJUZjYUJEkqKKX6Tx0REZ0j4vGI+Fs+v3JEPBwRL0XElRHRNV++eD7/cr5+UHv7tqEgSdLC7/vAcxXzZwC/TikNBiYDh+bLDwUmp5RWA36dl2uTDQVJkgpKKdV9ak9EDAR2Bc7P5wPYFrgmL3IJ2f2QAPbM58nXb5eXr8mGgiRJTSwihkbEoxXT0KoivwGOAWbn88sBU1JKs/L5UcCA/OcBwJsA+fqpefmavOpBkqSCFsRFDyml4cDw1tZFxG7AhJTSYxGxdcvi1nbTgXWtsqEgSdLC63PAHhGxC7AE0IMsYegVEV3y1GAgMCYvPwpYERgVEV2AnsCktg5g14MkSQXNTqnuU1tSSsenlAamlAYB+wN3p5S+CtwD7JsXOxC4If/5xnyefP3dqZ2BEDYUJEla9BwLHBURL5ONQbggX34BsFy+/Cg68N1Ndj1IklRQM92YMaV0L3Bv/vOrwKatlHkP2G9e9muiIEmSajJRkCSpoI7c52BhZ6IgSZJqMlGQJKmg2Yt+oGBDQZKkolLb9ypaJNj1IEmSajJRkCSpoBKMZTRRkCRJtZkoSJJUkJdHSpKkUjNRkCSpoDJcHmmiIEmSajJRkCSpIMcoSJKkUjNRkCSpoBIECiYKkiSpNhMFSZIKml2CSMFEQZIk1WSiIElSQSUIFEwUJElSbSYKkiQV5H0UJElSqZkoSJJUUAkCBRMFSZJUm4mCJEkFlWGMgg0FSZIKKkE7wa4HSZJUm4mCJEkFlaHrwURBkiTVZKIgSVJBJgqSJKnUTBQkSSqoBIGCiYIkSarNREGSpIIcoyBJkkrNREGSpIJKECiYKEiSpNpMFCRJKsgxCpIkqdRMFCRJKqgEgYKJgiRJqs1EQZKkghyjIEmSSs1EQZKkgkoQKJgoSJKk2kwUJEkqqAxjFGwoSJJUUAnaCXY9SJKk2kwUJEkqqAxdDyYKkiSpJhMFSZIKKkGgYKIgSZJqs6FQUjt8di2eHPFjnrnhZI4+eIe51nddrAt/Of1gnrnhZO6/9Gg+1X/Zj9cdfciOPHPDyTw54sdsv8VaHd6n1Iwu+8sl7L3nbuy1x6789dKL51h3yUUXsN6n12Dy5EkATJ8+ne9+59vst9ce7LXHrlw/4tpW9/mfZ59hnyG7s9tOO3D6acM+7seeOmUK3/rGwey+84586xsHM23qVCDr5z79tGHsttMO7LvX7jz3n2frd8Kar1JKdZ8azYZCCXXqFPzmuC+x5xHnssE+w9hvp41Yc5V+c5Q5aMgWTJ4+k3X2/Cm/v+weTv3+ngCsuUo/9vvihmy476nscfi5/Pb4L9GpU3Ron1KzeemlF7n2mqu57Iqrufq6G7j/vnt5/fXXABg3diwPPfgg/fuv8HH5Ky+/jFVWXZWrR9zIBRf/hV+deQYffvDBXPsd9rNT+MkpP+OmW+/gjddf4x8P3A/AhecPZ9PNtuCmW+9g08224ILzhwPwwN/v543XX+OmW+/gJ6f8nGE/O6Xu5y51lA2FEtpknUG88uZbvDb6bT6c9RFX3/5vdtt63TnK7Lb1ulx208MAXHfX42y96RofL7/69n/zwYezeH3M27zy5ltsss6gDu1Tajb/ffUV1l1vPbp160aXLl3YaONNuPuuOwH45Rm/4Ac//BER8XH5iODdGTNIKfHuuzPo2bMnnbvMOdRr4sQJzJjxDuutvwERwe57DOHukSMBuOeekewxZAgAewwZwj1335Utv3sku+8xhIhg3fXWZ/r0aUycOGFBPAT6H6VU/6nRFnhDISIOXtDH1JxW6NOTUeMnfzw/evxkBvTuOXeZcVmZjz6azbR3ZrJcr6UY0PuT5QCjJ0xmhT49O7RPqdmsttrqPPboo0yZMpmZM2fywN/vZ9y4cdx790j69O3DGmuuOUf5/f/fV3n11VfYfusvsO+QPTjm+BPp1GnOP6MTxo+nb99P0rS+/foxYcJ4ACa9/Ta9e/cBoHfvPkyalHVpTJgwnr79Krbp248J48fX5ZyledWIqx5+ClzUgOMqF8Rcy6obrZWfoj4uk4Aayzu1trxoBaUFZJVVV+XgQ7/Bt75xCEsuuSSrr7EGXTp35s/Dz+O8P184V/kHH3iANddci/MvupQ333iDb33zYDbcaGO6d+/+cZnW+pRb+52bQ2vbtPI7pebTDGMI6q0uiUJEPFVjehro28Z2QyPi0Yh4dPjw4fWomoDRE6YwsO8yH88P6LsMYyZOnbPM+CkM7JeV6dy5Ez26d2PS1BnZtv0qtu2zDGMnTu3QPqVmtPc++3HlNSO46NLL6NmzFysMGMDo0aP40t57svMO2zJ+/Dj233dv3po4kRuuv47tdtiRiOBTK63EgAED+e+rr86xv779+jF+/LiP58ePG0fvPlmKsOxyy33cpTBx4gSWXTYbJNynbz/Gj6vYZvwn20iNVq+uh77A14HdW5nerrVRSml4SmnjlNLGQ4cOrVPV9Oizr7Pap3qz0grLsViXzuz3xQ25+d6n5ihz831P89XdNwNg7+034L5HXsyW3/sU+31xQ7ou1oWVVliO1T7Vm0eeea1D+5Sa0dtvZ3+Sxo4Zw8i77mD3PYZw798f4tY77+bWO++mb99+XHHNdSzfuzf9+vfn4X8+lG331lu89tp/GbjiwDn217t3H5ZacimeevIJUkrcdOP1bLPtdgBsvc223Hj99QDceP31bLPNJ8tvuvF6Uko89eQTdO++9MddFGpuZbjqoV5dD38DuqeUnqheERH31umY6qCPPprND864ipvOPZzOnYJLbvgnz706jh8ftiv//s8b3Hzf01x8/YNcOOzrPHPDyUyeNoOvHZf1Fj336jiuveNxHr/2RGZ9NJsjT7+K2bMTkFrdp9Tsfnjkd5k6ZQpdunThhJNOpkfP2mNrhn77O/z4xOPZZ8jupJQ48qijWWaZLBX40t57ctV1NwBw4k9O4ccnHs/777/H5z6/JZ//wpYAHPKNofzoqCO5/rpr6Ne/P2ed/VsAvrDlVjxw/33stvMOLLFEN3427LQ6n7XUcdEMrZUaUrcNjmh0HaSmNPPxcwB4b1aDKyI1qSWyj8F1H+ixzkl31v1N9JlhOzR0wIqXR0qSpJr8rgdJkgpq4lR+vjFRkCRJNZkoSJJUUAkCBRMFSZJUm4mCJEkFZZeHL9pMFCRJUk0mCpIkFVSGMQo2FCRJKsjLIyVJUqmZKEiSVFAJAgUTBUmSVJuJgiRJBTlGQZIklZqJgiRJBZUgUDBRkCRJtZkoSJJUkGMUJElSqZkoSJJUkImCJEkqNRMFSZKKWvQDBRMFSZJUm4mCJEkFOUZBkiSVmomCJEkFmShIkqRSM1GQJKkgEwVJklRqJgqSJBVUhkTBhoIkSUUt+u0Eux4kSVJtJgqSJBVUhq4HEwVJklSTDQVJkgpKKdV9aktErBgR90TEcxHxbER8P1++bETcGREv5f8vky+PiPhdRLwcEU9FxIbtnaMNBUmSFl6zgB+mlNYCNgcOj4i1geOAkSmlwcDIfB5gZ2BwPg0F/tjeAWwoSJJUUKMThZTS2JTSv/OfpwPPAQOAPYFL8mKXAEPyn/cELk2ZfwK9IqJ/W8ewoSBJ0iIgIgYBGwAPA31TSmMha0wAffJiA4A3KzYblS+ryYaCJElFpfpPETE0Ih6tmIZWVyMiugPXAkemlKa1UeOocRY1eXmkJElNLKU0HBhea31ELEbWSLgspXRdvnh8RPRPKY3NuxYm5MtHAStWbD4QGNPW8U0UJEkqqNFjFCIigAuA51JKZ1esuhE4MP/5QOCGiuVfz69+2ByY2tJFUYuJgiRJC6/PAV8Dno6IJ/JlJwCnA1dFxKHAG8B++bpbgF2Al4F3gYPbO4ANBUmSCmr0nRlTSg/Q+rgDgO1aKZ+Aw+flGHY9SJKkmkwUJEkqqNGJwoJgoiBJkmoyUZAkqSATBUmSVGomCpIkFbXoBwomCpIkqTYTBUmSCirDGAUbCpIkFVSGhoJdD5IkqSYTBUmSCjJRkCRJpWaiIElSUYt+oGCiIEmSajNRkCSpIMcoSJKkUjNRkCSpIBMFSZJUaiYKkiQVZKIgSZJKzURBkqSCTBQkSVKpmShIklTUoh8omChIkqTaTBQkSSrIMQqSJKnUTBQkSSrIREGSJJWaiYIkSQWVIVGwoSBJUkFlaCjY9SBJkmoyUZAkqahFP1AwUZAkSbWZKEiSVJBjFCRJUqmZKEiSVJCJgiRJKjUTBUmSijJRkCRJZWaiIElSUWl2o2tQdyYKkiSpJhMFSZKKcoyCJEkqMxMFSZKKcoyCJEkqMxMFSZKKcoyCJEkqMxMFSZKKcoyCJEkqMxMFSZKKMlGQJEllZqIgSVJRJbjqwYaCJElFlaDroakbCjMfP6fRVZCa2hJN/RssaVHQzH9motEV0JwiYmhKaXij6yE1K39HSqgEXQ8OZtS8GNroCkhNzt8RLXKaOVGQJKm5lWCMgomCJEmqyURB88K+V6lt/o6UjWMUpE84SEtqm78jWhSZKEiSVJRjFCSIiJ0i4oWIeDkijmt0faRmEhEXRsSEiHim0XWR6sGGgtoUEZ2BPwA7A2sDX4mItRtbK6mpXAzs1OhKqEFSqv/UYDYU1J5NgZdTSq+mlD4ArgD2bHCdpKaRUrofmNToekj14hgFtWcA8GbF/ChgswbVRZKai2MUpFZvpd34LEyStECYKKg9o4AVK+YHAmMaVBdJai5NMIag3kwU1J5HgMERsXJEdAX2B25scJ0kSQuIDQW1KaU0CzgCuB14DrgqpfRsY2slNY+IuBx4CFgjIkZFxKGNrpMWoDS7/lOD2fWgdqWUbgFuaXQ9pGaUUvpKo+sg1ZMNBUmSiprtGAVJklRiJgqSJBXVBGMI6s2GgiRJRZWgoWDXgyRJqsmGgtRBEfFRRDwREc9ExNURseT/sK+tI+Jv+c97tPWtnBHRKyK+U+AYp0TE0R1dXlXm4ojYdx6ONchvT1Qp+aVQkirMTCmtn1JaB/gA+HblysjM8+9USunGlNLpbRTpBcxzQ0GS5gcbClIxfwdWyz9JPxcR5wL/BlaMiB0j4qGI+HeePHQHiIidIuL5iHgA2LtlRxFxUESck//cNyJGRMST+fRZ4HRg1TzN+GVe7kcR8UhEPBURP63Y14kR8UJE3AWs0d5JRMQ38/08GRHXVqUk20fE3yPixYjYLS/fOSJ+WXHsb/2vD6S0UCvBDZdsKEjzKCK6ADsDT+eL1gAuTSltAMwATgK2TyltCDwKHBURSwB/BnYHvgD0q7H73wH3pZTWAzYEngWOA17J04wfRcSOwGCyrwBfH9goIraMiI3IbrG9AVlDZJMOnM51KaVN8uM9B1TeVXAQsBWwK3Befg6HAlNTSpvk+/9mRKzcgeNIWkh51YPUcd0i4on8578DFwArAK+nlP6ZL98cWBv4R0QAdCW7ve+awH9TSi8BRMRfgaGtHGNb4OsAKaWPgKkRsUxVmR3z6fF8vjtZw2FpYERK6d38GB35To51ImIYWfdGd7Jbdbe4KqU0G3gpIl7Nz2FHYN2K8Qs982O/2IFjSYueJhhDUG82FKSOm5lSWr9yQd4YmFG5CLiz+ra+EbE+8+/ruQP4RUrpT1XHOLLAMS4GhqSUnoyIg4CtK9ZV7yvlx/5uSqmyQUFEDJrH40paSNj1IM1f/wQ+FxGrAUTEkhGxOvA8sHJErJqXq/X9ACOBw/JtO0dED2A6WVrQ4nbgkIqxDwMiog9wP7BXRHSLiKXJujnaszQwNiIWA75atW6/iOiU13kV4IX82Ifl5YmI1SNiqQ4cR1o0lWCMgomCNB+llCbmn8wvj4jF88UnpZRejIihwM0R8RbwALBOK7v4PjA8/wbCj4DDUkoPRcQ/8ssPb83HKawFPJQnGu8AB6SU/h0RVwJPAK+TdY+058fAw3n5p5mzQfICcB/QF/h2Sum9iDifbOzCvyM7+ERgSMceHUkLo0gl6F+RJKkeum1+bN3fRGf+84yo9zHaYteDJEmqya4HSZKKaoIxBPVmoiBJkmoyUZAkqagSjPMzUZAkSTWZKEiSVJRjFCRJUpmZKEiSVJRjFCRJUpmZKEiSVJRjFCRJUk0p1X9qR0TsFBEvRMTLEXHc/D5FGwqSJC2kIqIz8AdgZ2Bt4CsRsfb8PIZdD5IkFdX4rodNgZdTSq8CRMQVwJ7Af+bXAUwUJElaeA0A3qyYH5Uvm29MFCRJKmjm4+fU/SugI2IoMLRi0fCU0vCW1a1sMl+v2bShIElSE8sbBcNrrB4FrFgxPxAYMz+Pb9eDJEkLr0eAwRGxckR0BfYHbpyfBzBRkCRpIZVSmhURRwC3A52BC1NKz87PY0Qqwe0nJUlSMXY9SJKkmmwoSJKkmmwoSJKkmmwoSJKkmmwoSJKkmmwoSJKkmmwoSJKkmmwoSJKkmv4/yoPiYYab1CAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x648 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(9,9))\n",
    "sns.heatmap(cm_morg_test, annot=True, fmt=\".3f\", linewidths=.5, square = True, cmap = 'Blues_r');\n",
    "plt.ylabel('Actual label');\n",
    "plt.xlabel('Predicted label');\n",
    "all_sample_title = 'Accuracy Score: {0}'.format(score_morg_test)\n",
    "plt.title(all_sample_title, size = 15);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Validation set confusion matrix plotted**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgoAAAIBCAYAAAAoBpeWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xe8HFX9//HXJwkJwZCEkgJJ6CUERECq5UtvASkiCooCovmq8KN9QUBQQEEQUQEFJEixgoBEeg29hCZIR0IPkISSQiAQkpzfHzM3bDa7996M2dzNndczj3ncu7Nnz5zZ7N498z5nZiOlhCRJUi1dOroBkiSpedlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJdXXr6AZIkrSo+nAmDf8ehMW7EY3eRmtMFCRJUl0mCpIkFVSG71U0UZAkSXWZKEiSVFBq/BQF6NgpCiYKkiSpPhMFSZKKco6CJEkqMxMFSZIKKkGgYKIgSZLqM1GQJKkgr6MgSZJKzURBkqSCynAdBTsKkiQV5dCDJEkqMxMFSZIKKkGgYKIgSZLqM1GQJKkgT4+UJEmlZqIgSVJBC+f0yI5loiBJ0iIqItaMiMcqlqkRcWhEnBARr1esH17xmGMiYmxEPBcR27e5jVSGARZJkhpg0gezGv4hutQSXdt1xaWI6Aq8DmwC7A9MSymdXlVmGHAJsDGwPHArsEZKaVa9ek0UJEnqHLYGXkgpvdJKmV2BS1NKH6WUXgLGknUa6rKjIElS57AXWVrQ4qCIeDwiLoyIpfJ1g4DXKsqMy9fVZUdBkqQmFhEjIuLhimVEjTLdgV2Ay/NV5wKrAusBbwK/ailaYxOtDp/YUehgkXkpIlJErNbR7Wk2EbFsRPwuIl6MiA8j4o2IuCkiduvothUVEZ+PiAciYnr+f39wOx/3hYi4v+J5ODkiulWVuSN/LVUvi9ep81MRMS4vs07F+pXq1JMi4rn5qavi/u9GxPN5+x+JiK1rlBkUEaMiYlpEvJ3/3y9Ro9yKEXFJRLwbER9ExL8jYoeK+/tFxFkR8WBEzIiIl+u0+WsRcWVEvJm3e79a5fKyIyLiybz9EyLi7xX39Y6IE/PtTYmI8fl+rFGjnj4RcVFETMrL/jUilqkqU++5/6iq3NoRcXP+HLwdEedGRK+qMhfXqWtovX1V+6W0MJY0MqW0YcUyskZTdgT+lVKakLUrTUgpzUopzQbO55PhhXHAkIrHDQbeaG0fPT2y420GrJT/vhdwUsc1pblExGLA7cASwMnAC2Qv6u3IxuL+2XGtKybvDN4EXAscQ/bm/XVEfJBS+kMrj1sZuCV/7O7AasApwKeAQ6uK3w78qGrdR9R2LLX/DrxJ9tqs1BO4GbhhPusiIvYCfg+cANxDNtHq2ojYKKX0ZF6mG9n+zQC+BvQFfp3/3KeiriHA/cC/83reJztq6lmxyUF5HQ8AjwH967T5K2Tvv2uB79QpQ0ScBBxE9v58CBgAbF5RZAXgu8AF+fOwBNn/7wMRsW5KqTLq/TuwZr692cAvyF7LX6woU/3cA1wD3FvRpj7AbcB/8n1dBjgNWA6o7kg/S/ZcVXq53v5qkbQ3FcMOEbFcSunN/ObuwJP571cDf4uIX5NNZlwdeLDVmlNKLh24AL8FpgFjgKc6uj1VbVu8g7e/LVkktlGN+2IhbL9nA+o8j+wPe7eKdeeQjRnW3af8cS9WPe5g4GNguYp1dwBXtLMtq+Wvve/lz/M6bZT/al5uk/mtC3gOuLDidhfgCeAvFev2BmYBK1dtczawesW6S4G7gS6ttLVLxe+nAy+3Vg7olbd7vxpl1s7btW0r2/tU9esFWDp/To6vWLdZvp3/qVi3cb5um1bqbynztYp1xwBTgb4V63bJy21Yse5i4OFGvEdcEm9P+zg1emmrDWQd03eAPhXr/py/xx4n6xxU/p04luzA6zlgx7bqd+ihA0V2KsueZP+JFwLDImLdGuVaYta384jx8Yj4esX9PSPitIh4JSI+yuPsUyruTxFxUFWdJ0TE2xW398vLbZzH19OBI/P7To2IJ/I4eFwelQ6s0c7v5uVaotkr8ph1p4iYnR8VV5ZfOV+/S52nqG/+c3z1HSl/tVfUtW5EXBMRk/N2PhgR21Zt65+RnWP8Xl52tao6UkQcHhFnRMRbZG+ylvt2jWxs8MM8Vj4tTzzm147AlSmlmRXrLiVLSuaJ6yusB9xR9bibyY7gtyvQDoAzgD+QHW22x17ASymlB+anrohYBVgDuKxlXcri0MvJno8WOwIPpWwmdot/kiUMO+R19QG+DJyT11FTa/cVKLcv2UzyW1qp5/2U0vSqde8CrzB3mrEjMCGldFdFuQeBl5j7uai2F1lyck3FuvXIOgCTK9bdTNZR2KnVPVKnklL6IKW0TEppSsW6b6aUPp1SWjeltEv6JF0gpXRySmnVlNKaKaV6CeEcdhQ61lZkEealwBVkR4d7VxaIiP5kMetGwBHAl8jizSH5/QFcBXwfOBsYDhwPLFuwTZeQxbDD85+Q/aH7Odkfn0OBVYDb8o5OSzuPIzvqvZMs9vw+MIXsSO1GsjGwfau2tR/wFnB9nbY8RnY0eWFk4/P1Yu2hZJHscmRHtLsDo/jkOeoBjAbWIouH9wNWBu6MiKWrqjsyr+ebZEfsRMRXgSvJ4rldgBOBEWTRf0sbWsb096uzL0TEp/I2VX+YPpP/bG3MeHGyD8xKLcMJa1Wt3y7vUH4Q2XyOWp3P4cCm+b60KSJ6k32QXVLjvrbqatmvWvu9dET0qyg3V5mU0gyyI5+WOjYAFgNSRNwbER/nnddj8vdCI2wCPNHSuc4747dGRPXzPpd8v1YDnq5YPc8+5p6hzv9/vl97AlellD6ouKvWa2Im2Xumum3D8k7yRxFxT0RsjhaItBDmKHQ05yh0rL2BycCNKaUZEXELsFdE/KjiiPkwoA/w2Yoe4eiKOrYji+h3TSldXbH+TwXbdFZK6czKFSmlb7f8nncO7iebEPN54K6I6Es2Jn5GSunwiodeWfG4i4F9I+LElFLK//jtC/y56ii5crvPR8SRwKlkUfOHEXEncEFK6fKKoseTdUq+WHFUV3n0tz/ZGPIaKaUX8/Y8QBbl/y8VH/jA+JTS1yraHcAvgT+llH5Qsf4j4OyIOCWl9A7ZUdwssj/S9bQkJJOr1k/Kfy5FfWPJOouVWiYnVXZ27gT+mJdfkSxivDsiPpNSejlve3fgTOAnKaVJ7fx83Y3sg+nSypXtrKtlv1rb77fyn9VlWsq11NGSZJ1HNqv7OGBLsrkDU8iGcRa0gWQdlJaO5sx8ezdGxJoppQ/rPO5XZEMPlc9Za/u4Sp16vkiWOF1atX4s8PWIWCyl9HG+7rNAV+Z+TTxKNlfjaaAf8H/ALRHxhTzNkFplotBB8qPc3YFR+VETZEdrK5EdnbXYiqwj8Sa1bQW8W9VJ+G9cV6OtO0bEfRExheyP5Lj8rpYZ3ZuRTSS7qJV6LyT74Noiv71lfru1x5BS+jXZ0f+BZLHrJsBllUMrZM/B36uj3wobk80GfrGi3nFkKcQXqspW7/8aZJ2MyyKiW8tCNolscfLhgpTSKymlbiml9nTQ6h0jtHbscC6wQUT8OLIzQTYl60DNypeW/To+pXRRSunulNJfyJ7nxNwTHg8HPiT7sG2vvcnm0DxRtX5+6qrev6ixvtZzEBXrW/5m3ZBSOjqldHtK6SdknaNj2tGGIrqQzUHYI6U0KqV0Ddl7dxDwjVoPiIjvk03A/E7ekazU1j5W25usI3FT1frzyT74fxsRAyNibbKOUvVr4syU0rkppTtTSleQvV9eZ94JryogLYSlo9lR6Dg7kh1hXh8RffOj8jvI4uTK4YdlyGag19PW/fNrQuWNiNiIbA7FOLI4fjM+6ci0nHLXcmpX3XbkH9J38MnM6/2BB1NKT7XVoJTS6ymlc1JKXyU7sroRODI+OaWsredguer9yk1g7iMvapRrGcK5nmxoqGVpGUcfQvu1HEn2rVpf74h7jpTSrWRHz8eSHX3fRTYE9W6NNlc+bjxZh2gDmBOHH0t29sGS+euu5XS6JfPhkbnkz/M2VA07zEddLclB9X5XJyyTapRpKddS5t385+1VZW4DBudDJAvau2TzClqGiFpezy8Dw6oL53NufgsclVIaVXV3e/axsq5uwB7APyoOKFra8CzZENjeZK//x8mGxx6j9dfEdLLX8wb1ykiV7Ch0nJbOwOVkfzwmkc187wF8tWL8/x2yD7p62rofss5H96p11R+QLao7sLuTfTB9LaV0dUppDPNOLmw5YmqrHX8A9oiIQWQT0lpNE2o2LqX3yY6aupKN/7Zsv7Vtv0nt0+MG8MkHz5xNVN1uuX8EWfRfvbQ5Eaiq7a8x71h0vTH86sefTNZxWTdv++/JjijHtGfz+c9BZB/mV/DJ665lgtx9ZPNdqn2FbJiyOvpub10t+1Vrv99NKb1VUW6uMvnQxioVdTxDbS3pRLsmMc6n1rY51/Yi4nNkz9PvU0q/rPGYefYxV2/uwtZk/8fzzA0BSCldSPZaWJfsVLeDyN4X8/Oa0H+jBJGCHYUOENkFUXYme/NvWbUcTvbG3zIvPhrYPiIG1KluNNmEsJ1b2eQ4KiY3RUQXsvixPXoCH1edZVAdt94PTGfeyYrVriSbfHUp2Wuv+oNnLhGxdJ0JjKvnPyfmP0eTda5qXlSIbHz2s1Fx1kXeWfkc2Tn9rXmOLKZdKaX0cI2lOlZuyw3A7pUTQcnOgX+NT85zriulNC2l9ERKaRLZcMwrZF/qUlP+uvk88Ei+aizzvuYOy+/7Ntn4dbW9ydKfF6rWt6uu/Oj7P2QT8lra1SW/XdnRugHYKCJWrFi3C1nn+ca8rpeBp8g+QCu1XON+Wo32/7euBQZG9mU6AETEqmRDZ/+uWLd2XvZG8omwNdyQ1/WFisdtSNYZqtXp3JusY35HvcallD7MXxMTyIY7ulBxhkm1iOhJlmg+Uq+M2i8thH8drtY5ky6NXcg+aBO1z0dfDHibbMIeZEcT48j+0O5L9gF/IPDD/P4g+8M0leysiK3y+s+rqPOXwAfAD8hOM7sCeBV4u6LMfnmbelW1Z3i+/gyyP8Y/JvvwTMBBFeWOITu6Oivfxq7ASGBQVX2/yx/7t3Y8T18BnicbS922YvvvA9dUlFsz3/8HyT50tyE7e+Hb+f09yCYuPkt2Xv4eZKc+vg4sXVHPXPtUsf5rZB2c3+bPxzZkCcP1wBJ5mRXJ5m98q419arnewN/IPlh/SDaU8Z2qcjPJJghWPu4n+XO7M1maMIOKc/vJjiqvy/8vt8xfL8+SpSIrtNKmLfJ9n+c6CmRHqbOAQ9v52q5ZF59cI6Fl8uHFZJ3LdSrKLEbWWXokf55bPiT/UlXX7vlr7Zf56+LkvO5v1Hj9fIVs6Gxixe1+FWWG5ev2ydv9u/z25hVluuZteja/b/f89fM0sFhepj9ZZ+/V/DnYtGIZVtWuG/PX45fJJok+B9xd47nsQTYccUad57o32cWadgK2J5uz8jEV14Igmwh9N9mk3a3JXstjyFLGDWvV6zJ/y4SpM1Kjl47exw5/ksu4kB11/KeV+88hi3F75LdXJLua2ySyD/x/A3tVlO9JdlGZcfkfgJeAkyvu70U22evd/A/vcWTjym12FPL7fpj/EXyf7Oh1dWp8qOZ/jJ7O2zCe7Kimd1WZbfLH1r24TEXZIfl+PZb/wXwv/wN9DPkHdEXZdck+uN/LlweArSvuX4XsnPz3yD6or6XiIj55mZodhfy+HfM/uO+TdUoeI5v53i2/f6X88fu1Y7++QNap+ZBsnPvgGmUScELF7RXI5iVMydtwB9lZHpWPGZQ/B2+SdSLeAf4BDG2jPVtQv6NwKNmH8PLtfG23Vtd3yVKIj4B/Vf7/VJQZnP8/Tcvbf3b1/3Vebh+yIYEZeZ3fq/Mc1lq2qChzQp0yd1TVNYAsAZySv4ZGUdH5qtjv9tTVl2zYbXL+WvobsGyN9u+WP37TOs/1p8ium/AuWafrIWC3qjKLkyV5r+XP+xSyjkrNOl3mfxk/ZUZq9NLR+xj5i0laKCLiNLKjmpVTOy+KI0nNasLUjxv+ITqg92KNukZIu3gdBS0UEbEmWcz7feBEOwmSOoMyHGrbUdDCch7ZNRCuJpvHIElaBNhR0EKRUtqio9sgSQtcCSIFT4+UJEl1mShIklRQKkGk0Mwdhc7/7EuSGqlDzxboLJq5o8C0j+wrSLX06pH9/es5/Mw2SkrlNP36QxbKdspwhQHnKEiSpLqaOlGQJKmZlSBQMFGQJEn1mShIklSQcxQkSVKpmShIklRY548UTBQkSVJdJgqSJBXkHAVJklRqJgqSJBVUgkDBjoIkSUU59CBJkkrNREGSpILK8DXTJgqSJKkuEwVJkorq/IGCiYIkSarPREGSpIJKECiYKEiSpPpMFCRJKsjrKEiSpFIzUZAkqSCvoyBJkkrNREGSpKI6f6BgoiBJkuozUZAkqaASBAomCpIkqT4TBUmSCvI6CpIkqdRMFCRJKsjrKEiSpFIzUZAkqajOHyiYKEiSpPpMFCRJKqgEgYIdBUmSivL0SEmSVGomCpIkFeTpkZIkqdRMFCRJKqrzBwomCpIkqT4TBUmSCipBoGCiIEmS6jNRkCSpIK+jIEmSSs1EQZKkgryOgiRJKjUTBUmSiur8gYKJgiRJqs9EQZKkgkoQKJgoSJKk+kwUJEkqyOsoSJKkUjNRkCSpIK+jIEmSSs1EQZKkojp/oGBHQZKkokrQT3DoQZIk1WeiIElSQZ4eKUmSSs1EQZKkgjw9UpIklZqJgiRJRXX+QMFEQZIk1WeiIElSQSUIFEwUJElSfSYKkiQV5HUUJElSqZkoSJJUkNdRkCRJpWZHQZKkotJCWNoQEX0j4oqIeDYinomIzSJi6Yi4JSKez38ulZeNiDgrIsZGxOMRsUFb9dtRkCRp0XYmcGNKaSjwGeAZ4GhgdEppdWB0fhtgR2D1fBkBnNtW5XYUJEkqqKMDhYjoDfwPcAFASmlGSmkysCvwx7zYH4Hd8t93Bf6UMmOAvhGxXGvbsKMgSdKiaxXgLeCiiHg0Iv4QEZ8CBqSU3gTIf/bPyw8CXqt4/Lh8XV12FCRJKiilxi8RMSIiHq5YRlQ0oRuwAXBuSml94H0+GWaoJWrtRmv76OmRkiQ1sZTSSGBknbvHAeNSSg/kt68g6yhMiIjlUkpv5kMLEyvKD6l4/GDgjda2b6IgSVJBaSH8a3X7KY0HXouINfNVWwNPA1cD++br9gWuyn+/GvhWfvbDpsCUliGKekwUJEkqqjmut/T/gL9GRHfgRWB/siDgsog4AHgV2DMvez0wHBgLfJCXbZUdBUmSFmEppceADWvctXWNsgk4cH7qt6MgSVJBzREoNJZzFCRJUl0mCpIkFTS7BN8zbaIgSZLqMlGQJKmgzp8nmChIkqRWmChIklRQCaYomChIkqT6TBQkSSqorUssdwYmCpIkqS4TBUmSCprd+QMFEwVJklSfiYIkSQU5R0GSJJWaiYIkSQV5HQVJklRqJgqSJBVUhjkKdhQ6sZdfepFjfnj4nNuvj3uN7/3gYB5//DFeefklAN57bypLLtmbSy7/J2Puv5ffnvErPv74YxZbbDEOOfyHbLzJpvPUO2XKZI458nDeeON1ll9+EKee/ht69+5DSolf/uJk7r37LhZffHFO+NkprDVsbQCuuWoUF5z/ewAO+O73+NKuuy+EZ0Bqv9UH9eXPRw+fc3vl5Xrzsz+P4XdXPQbAoV/egFO+80UG73Ue70z9cJ7Hf2PrtTh6r40BOPXSB/nr6GcAWH+1/ow8fFt6du/GTQ+9zP+ddycAS/XqwZ+PGc6K/XvzysSp7HPK9Uye9lGjd1Oab3YUOrGVVl6FSy7/JwCzZs1ix202Z8utt+Hr39x3Tplfn34qvXotCUDfvktxxm/PpV//AYx9/j8c9P3vcOOtd81T78UXnM9Gm2zK/geM4KILRnLxBedz8GFHcO89d/HaK6/wz2tv4snH/80pJ53In/52GVOmTOb835/Nny+9gohgn6/tweZbbkXv3n0WzhMhtcPzr09m0//3NwC6dAle+NMBXH3/CwAMXrYXW62/Aq9OnFrzsUv16sGxX9+Ezx9yCQm478y9ue6BF5k87SPOOnBLDjprNA88O55//nRXtttwRW5++BWO+OqG3PHYa5x++cMcseeGHLHnhhx30b0La3e1gHgdhf9CRAyNiKMi4qyIODP/fa1GbU+te/CB+xk8ZAjLLT9ozrqUErfedCM77LgTAEPXGka//gMAWHW11Znx0UfMmDFjnrruvH00O++yGwA777Ibd9x265z1O31pVyKCT39mPaa9N5W33prI/ffewyabfY4+ffrSu3cfNtnsc9x3z92N3mWpsC0/M4SXxk/h1YnvAXDaiP/h2AvvqTtxbdvPrsjoR19l0rSPmDztI0Y/+irbfXZFBi61BEsu0Z0Hnh0PwN9GP8OXNl0VgJ03XZW/3Po0AH+59Wm+tNmqjd8xLXBpIfzraA3pKETEUcClQAAPAg/lv18SEUc3Yptq3c03Xs/2eYegxaOPPMzSyyzDCiuuNE/50bfcxJpDh9G9e/d57nvn3Xfo168/AP369efdd98FYOLECQwYuNyccv0HDOStiRNqrp84ccKC2C2pIfbcfA0uu+M5AHbaZGXeeGcaT7z0dt3yyy/Ti3Fvvzfn9uvvTGP5ZXqx/LK9eP3taZ+sf3sayy/bC4D+fZdg/KQPABg/6QP69enZiF2R/muNGno4AFg7pfRx5cqI+DXwFHBqg7arGj7+eAZ33nEbBx1y+Fzrb7zhunk6DwAvjH2es874FWefd8F8bafW0VZEUKtDHBHzVbe0sCzWrQs7bbIKP7n4Pnr26MZRe23MzseOavUxtV7OKSWCee9IZTifrkTK8N/ZqKGH2cDyNdYvl99XU0SMiIiHI+LhkSNHNqhp5XPvPXczdK1hLLPMsnPWzZw5k9tH38J22w+fq+yE8eM54rCD+OnJv2DIkBVq1rfM0svw1lsTAXjrrYksvfTSAAwYMIAJ49+cU27ihPEs268//Wusb0kkpGaz/YYr8dgLE5k4+QNWWa4PKw7ozYNnf4NnL9qfQcv24v6zvs6ApZaY6zGvvz2NwcsuOef2oGV68ea77/P62+8xKE8QAAYt24s333kfgImTP2BgXs/ApZbgrSnTF8LeSfOvUR2FQ4HREXFDRIzMlxuB0cAh9R6UUhqZUtowpbThiBEjGtS08rnphuvmzENo8eCY+1lp5ZUZMHDgnHXvTZ3KIQf9LwcdfDjrrb9B3fr+Z4utuPbqbJLktVf/k8233HrO+uuuuYqUEk/8+zF6Lbkk/fr1Z7PPf4Ex993L1KlTmDp1CmPuu5fNPv+FBuyp9N/76uZrcNmd/wHgqZffYcWvn8/Q/S9i6P4X8frb09js4L8xIR8yaHHLI6+wzQYr0LdXD/r26sE2G6zALY+8wvhJHzBt+sdsvGb2Pvv61mtx7ZgXAbhuzIvss80wAPbZZhjXjnlhIe6lFpSUGr90tIYMPaSUboyINYCNgUFk8xPGAQ+llGY1Ypuqbfr06Txw/7386McnzrX+phuvY/sdd55r3d8v/Suvvfoqfxh5Ln8YeS4AZ//+ApZeZhl+evxxfOWrX2PY2p9mvwO+y9FHHMZVo/7BwIHL8YtfnQHAF764OffefRe77rRdfnrkzwHo06cv3/nfH/DNvfcE4Lvf+wF9+vRt9K5L861nj25stf4KHPTb29osu8Hq/fnO8E/zgzNHM2naR5xyyYPcc8ZeAPz8kgeZlJ/qePDZtzHysG3p2aMbNz/8Cjc9/DIAp1/+MH85Zjj7brc2r731Ht/4+XUN2y/pvxFNPF6Wpn3UtG2TOlSvHtnYd8/hZ3ZwS6TmNP36Q4Aak0QWsOufmtjwD6rha/fv0EldXsJZkiTV5QWXJEkqqHlD+QXHREGSJNVloiBJUkHNcOXERjNRkCRJdZkoSJJUkHMUJElSqZkoSJJU0GznKEiSpDIzUZAkqSDnKEiSpFIzUZAkqaASBAomCpIkqT4TBUmSCmrib2BeYEwUJElSXSYKkiQVNLujG7AQ2FGQJKkghx4kSVKpmShIklRQ588TTBQkSVIrTBQkSSrIOQqSJKnUTBQkSSqoDKdHmihIkqS6TBQkSSrIOQqSJKnUTBQkSSqoBIGCiYIkSarPREGSpIJKECiYKEiSpPpMFCRJKmh2CSYpmChIkqS6TBQkSSqo8+cJJgqSJKkVJgqSJBXklRklSVKpmShIklRQGb490o6CJEkFlWDkwaEHSZJUn4mCJEkFecElSZJUaiYKkiQVVIJAwURBkiTVZ6IgSVJBzlGQJEmlZqIgSVJBszt/oGCiIEmS6jNRkCSpoBJMUTBRkCRJ9ZkoSJJU0Gw6f6RgoiBJkuoyUZAkqSDnKEiSpFIzUZAkqSCvoyBJkkrNREGSpIL8rgdJklRqJgqSJBVUgkDBREGSpKJmp8Yv7RERXSPi0Yi4Nr99cUS8FBGP5ct6+fqIiLMiYmxEPB4RG7RVt4mCJEmLvkOAZ4DeFeuOTCldUVVuR2D1fNkEODf/WZeJgiRJBaWUGr60JSIGAzsBf2hHk3cF/pQyY4C+EbFcaw+woyBJ0qLtDOCHwOyq9Sfnwwu/iYge+bpBwGsVZcbl6+qyoyBJUkELY45CRIyIiIcrlhEt24+InYGJKaVHqpp2DDAU2AhYGjiq5SE1dqPV2MI5CpIkNbGU0khgZJ27Pw/sEhHDgcWB3hHxl5TSPvn9H0XERcAR+e1xwJCKxw8G3mht+yYKkiQV1NFnPaSUjkkpDU4prQTsBdyWUtqnZd5BRASwG/Bk/pCrgW/lZz9sCkxJKb3Z2jZMFCRJ6nz+GhH9yIYaHgO+l6+/HhgOjAU+APZvqyI7CpIkFZRaH95fqFJKdwB35L9vVadMAg6cn3odepAkSXXVTRQiYunWHphSenfBN0eSpEVHGb5murWhh0fITpmodyrFKg1pkSSahZMnAAAXjUlEQVRJahp1OwoppZUXZkMkSVrU+KVQzPkCiX0i4sf57RUiYuPGN02SJHW09pz1cA7ZZSG3An4GvAf8g+xqT5IkldbsEkQK7ekobJJS2iAiHgVIKU2KiO4NbpckSWoC7ekofBwRXcmvBZ1fwKH6iyckSSqdMpz10J7rKJwFjAIGRMTJwD3AzxvaKkmS1BTaTBRSSn+NiEeArfNVu6WUnmlssyRJan4lmKLQ7ks4LwG0DD/0bFxzJElSM2nP6ZE/Af5I9n3WywIXRcRxjW6YJEnNbnZKDV86WnsShb2B9VNKHwJExKnAv4CTGtkwSZKaXRN8jjdceyYzvgwsXnG7B/BCQ1ojSZKaSmtfCvVbsjkJHwFPRcQt+e1tyc58kCSp1MpwrYDWhh4ezn8+QnZ6ZIs7GtYaSZLUVFr7Uqg/LsyGSJK0qGmGyYaN1uZkxohYHTgFGEbFXIWUkl8zLUlSJ9eesx4uAo4HfgNsCewPRCMbJUnSoqAEgUK7znromVIaDURK6ZWU0glk3yQpSZI6ufYkCh9GRBfg+Yg4CHgd6N/YZkmS1Pz8UqjMoWSXcD4Y+CzwTWDfRjZKkiQ1h/Z8KdRD+a/TyOYnSJIkIJVgkkJrF1y6huwCSzWllHZpSIskSVLTaC1ROH2htUKSpEVQGeYotHbBpTsXZkMkSVLzac9ZD5IkqYYyJArtOetBkiSVVFMnCr16eAFIqTXTrz+ko5sglZpnPXjWgyRJpdbUZz30XP+gjm6C1JSmP/o7AD6c2cENkZrU4gspL5+9cDbToTzrQZIk1eXXTEuSVFAZ5ii056yHi4BzgZlkXzP9J+DPjWyUJElqDn7NtCRJBaXU+KWj+TXTkiQVNLsZPskbzK+ZliRJdfk105IkFVSCQKFdZz3cTo0LL6WUnKcgSVIn1545CkdU/L44sAfZGRCSJJVaGU6PbM/QwyNVq+6NCC/GJElSCbRn6GHpiptdyCY0DmxYiyRJWkSUIFBo19DDI2RzFIJsyOEl4IBGNkqSJDWH9nQU1kopfVi5IiJ6NKg9kiQtMryOQua+GuvuX9ANkSRJzaduohARA4FBQM+IWJ9s6AGgN9kFmCRJKrXOnye0PvSwPbAfMBj4FZ90FKYCP2pssyRJUjOo21FIKf0R+GNE7JFS+sdCbJMkSYuEMlxHoT1zFD4bEX1bbkTEUhFxUgPbJEmSmkR7Ogo7ppQmt9xIKU0ChjeuSZIkLRpmp8YvHa09HYWuladDRkRPwNMjJUkqgfZcR+EvwOiIuIhsgue3gT81tFWSJC0CyjBHoT3f9XBaRDwObEN25sPPUko3NbxlkiSpw7UnUSCldCNwI0BEfD4izk4pHdjQlkmS1ORKECi0r6MQEesBewNfI/uuhysb2ShJktQcWrsy4xrAXmQdhHeAvwORUtpyIbVNkqSmVvY5Cs8CdwNfSimNBYiIwxZKqyRJWgQ0w+mLjdba6ZF7AOOB2yPi/IjYmk8u4yxJkkqgtUs4jwJGRcSngN2Aw4ABEXEuMCqldPNCaqMkSU2pDEMPbV5wKaX0fkrprymlncm+IOox4OiGt0ySJHW49lyZcY6U0rsppfNSSls1qkGSJC0q0kJYOtp8dRQkSVK5tOs6CpIkaV6znaMgSZLKzERBkqSCShAomChIkqT6TBQkSSrI6yhIkqRSM1GQJKmgEgQKJgqSJKk+EwVJkgryOgqSJKnUTBQkSSqoBIGCiYIkSarPREGSpIK8joIkSSo1EwVJkgqa3fkDBTsKkiQVlej8PQWHHiRJUl0mCpIkFVSCuYwmCpIkqT4TBUmSCvL0SEmSVGomCpIkFVSG0yNNFCRJWkRFxOIR8WBE/DsinoqIE/P1K0fEAxHxfET8PSK65+t75LfH5vev1NY27ChIklRQSqnhSxs+ArZKKX0GWA/YISI2BX4B/CaltDowCTggL38AMCmltBrwm7xcq+woSJK0iEqZafnNxfIlAVsBV+Tr/wjslv++a36b/P6tIyJa24YdBUmSCkqp8UtbIqJrRDwGTARuAV4AJqeUZuZFxgGD8t8HAa9lbU8zgSnAMq3Vb0dBkqQmFhEjIuLhimVE5f0ppVkppfWAwcDGwFo1qmnpctRKD1rtjnjWgyRJBc1eCNdRSCmNBEa2o9zkiLgD2BToGxHd8tRgMPBGXmwcMAQYFxHdgD7Au63Va6IgSdIiKiL6RUTf/PeewDbAM8DtwFfyYvsCV+W/X53fJr//ttTGjEkTBUmSCmqCCzMuB/wxIrqSHfxfllK6NiKeBi6NiJOAR4EL8vIXAH+OiLFkScJebW3AjoIkSYuolNLjwPo11r9INl+hev2HwJ7zsw07CpIkFeR3PUiSpFIzUZAkqaASBAomCpIkqT4TBUmSCirDHAU7CpIkFVSCfoJDD5IkqT4TBUmSCirD0IOJgiRJqstEQZKkgkwUJElSqZkoSJJUUAkCBRMFSZJUn4mCJEkFOUdBkiSVmomCJEkFlSBQMFGQJEn1mShIklSQcxQkSVKpmShIklRQCQIFEwVJklSfiYIkSQU5R0GSJJWaiYIkSQWVIFAwUZAkSfWZKEiSVFAZ5ijYUZAkqaAS9BMcepAkSfWZKEiSVFAZhh5MFCRJUl0mCpIkFVSCQMFEQZIk1WeiUFLbfm4tTj/yK3Tt0oWL/3kfp190y1z3d1+sGxf87Jusv9YKvDvlffY56kJeffNdAI749nbst+tmzJo9m/877Qpuvf+ZdtUpNZuXX3qRH/7fYXNujxv3Gj846GA23GgTTvrp8cz46CO6duvKj447gU+vuy5Tp0zhJz/+EeNee5Xu3Xtw4kk/Z/XV15in3nHjXuOoIw5n6pQpDB02jJ+fchqLde/OjBkzOPaYH/LMU0/Rp29fTvvVbxg0aDAAF5x/HqP+cQVdunbhqGOO4/Nf+OJCex5UnHMU1Cl16RKccfRX2fWgc1h/j5PYc4fPMnSVgXOV2W+3zZj03nTW2fVEfvvX2zn5kF0BGLrKQPbcfgM2+MrJ7HLgOZx5zFfp0iXaVafUbFZaeRUuu/IqLrvyKi65/EoWX7wnW22zLb/59S/53g8O5LIrr+IHBx3CGb/+JQB/OP/3DB26FleMuoaTT/kFp51ycs16z/z16ezzrf245oab6d27N6OuvAKAUf+4nN69e3Ptjbewz7f244xfnw7AC2PHcuP113Hl1ddxznl/4OcnncisWbMWzpMgtcGOQglttM5KvPDa27z8+jt8PHMWl9/0L3beYt25yuy8xbr89ZoHALjy1kfZYuM156y//KZ/MePjmbzyxju88NrbbLTOSu2qU2pmD4y5nyFDhrD88oMIgmnT3gdg2nvv0a9ffwBefOEFNt5kUwBWXmVV3njjdd55++256kkp8eADY9h2u+0B2GXX3blt9GgAbr/tNnbZdXcAtt1uex4ccz8pJe64fTQ7DN+J7t27M3jwEIYMWZEnn3h8oey3/jspNX7paAu9oxAR+y/sbWpuy/fvw7gJk+bcfn3CJAb16zNvmfFZmVmzZjN12nSW6fspBvX7ZD3A6xMnsXz/Pu2qU2pmN95wHTsM3xmAHx79I35z+mlst/Xm/Or0X3DwYYcDsMaaQxl9azak9sTjj/PmG28wYcL4ueqZPHkSSy7Zm27dspHdAQMGMnHiBAAmTpzAwIHLAdCtWzd6LbkkkydPYsKECQwY+EkCN2DgACZOmNDYHZbaqSMShRM7YJuqEMQ866o7rRE1yiSgzvr21Ck1q49nzODO229ju+13AOCyv1/CkUcdw82j7+TIo47hhB8fC8C3vzOCqVOn8tUv78olf/szQ4euRdeuc0/1qnUE2PJ+qjWeHRE1H1TrPajmk1Jq+NLRGjKZMSLqZWYBDGjlcSOAEQDnnXdeA1omgNcnTmbwgKXm3B40YCneeGvK3GUmTGbwwKV4feJkunbtQu9ePXl3yvvZYwdWPLb/UryZP7atOqVmdc89dzF02Noss+yyAFxz1SiOOibrHGy3/Y6c+JPjAOjVqxc/O/kUIPuAGL7d1gwaPHiuupZaainee28qM2fOpFu3bkyYMH7O0MWAAQMZP/5NBgwcyMyZM5n23nv06dOXAQMHMmH8J8nEhPET6Ne/f8P3W2qPRiUKA4BvAV+qsbxT70EppZEppQ1TShuOGDGiQU3Tw0+9wmor9GPF5ZdhsW5d2XP7Dbjujrn7dtfd+QTf+NImAHx5m/W586H/ZOvveJw9t9+A7ot1Y8Xll2G1Ffrx0JMvt6tOqVndcP117Dh8pzm3+/Xvz8MPPQjAgw+MYYUVVwJg6tSpfDxjBgBXXnE5G2y4Ib169Zqrrohgo4034ZabbwLg6qtGseVWWwGwxZZbcfVVowC45eab2HiTTYkINt9yK268/jpmzJjBuHGv8eqrL7POp53jsygwUSjuWqBXSumx6jsi4o4GbVPtNGvWbA77xWVcc86BdO0S/PGqMTzz4nh+/P2d+NfTr3LdnU9w8T/v48KTvsWTVx3PpKnv882jLwLgmRfH84+bH+XRfxzLzFmzOfTUy5g9OwGpZp1Ss5s+fTpj7ruPHx//0znrfnLCzzjt1J8za+ZMuvfowU9OyO576cUXOO6Yo+jStQurrLoaJ/70k7MeDvzedzn+pyfRv/8ADj38SH54xGGcfdYZDF1rLXbfY08Adt/jKxx79JHsvMO29O7Th9NO/w0Aq622OtvtsCO77zKcrl278qPjfkLXrl0X4rMg1RfN0FupI/Vc/6COboPUlKY/+jsAPpzZwQ2RmtTi2WFwwyd6rHPcLQ3/EH3ypG07dMKKp0dKkqS6vDKjJEkFNXEqv8CYKEiSpLpMFCRJKqgEgYKJgiRJqs9EQZKkgrLTwzs3EwVJklSXiYIkSQWVYY6CHQVJkgry9EhJklRqJgqSJBVUgkDBREGSJNVnoiBJUkHOUZAkSaVmoiBJUkElCBRMFCRJUn0mCpIkFeQcBUmSVGomCpIkFWSiIEmSSs1EQZKkojp/oGCiIEmS6jNRkCSpIOcoSJKkUjNRkCSpIBMFSZJUaiYKkiQVZKIgSZJKzURBkqSCypAo2FGQJKmozt9PcOhBkiTVZ6IgSVJBZRh6MFGQJEl1mShIklSQiYIkSSo1EwVJkgoyUZAkSaVmoiBJUlGdP1AwUZAkSfWZKEiSVJBzFCRJUqmZKEiSVJCJgiRJKjU7CpIkFZRSavjSloi4MCImRsSTFetOiIjXI+KxfBlecd8xETE2Ip6LiO3bqt+OgiRJi7aLgR1qrP9NSmm9fLkeICKGAXsBa+ePOSciurZWuR0FSZIKaoZEIaV0F/BuO5u8K3BpSumjlNJLwFhg49YeYEdBkqTO6aCIeDwfmlgqXzcIeK2izLh8XV12FCRJKio1fomIERHxcMUyoh0tOxdYFVgPeBP4Vb4+6uxFXZ4eKUlSE0spjQRGzudjJrT8HhHnA9fmN8cBQyqKDgbeaK0uEwVJkgpqhjkKtUTEchU3dwdazoi4GtgrInpExMrA6sCDrdVloiBJUkHNcMGliLgE2AJYNiLGAccDW0TEemTDCi8D/wuQUnoqIi4DngZmAgemlGa1Vr8dBUmSFmEppb1rrL6glfInAye3t347CpIkFdQMiUKjOUdBkiTVZaIgSVJRnT9QMFGQJEn1mShIklSQcxQkSVKpmShIklSQiYIkSSo1EwVJkgoyUZAkSaVmoiBJUkEmCpIkqdRMFCRJKqrzBwomCpIkqT4TBUmSCnKOgiRJKjUTBUmSCjJRkCRJpWaiIElSQWVIFOwoSJJUUBk6Cg49SJKkukwUJEkqqvMHCiYKkiSpPhMFSZIKco6CJEkqNRMFSZIKMlGQJEmlZqIgSVJRJgqSJKnMTBQkSSoqze7oFjSciYIkSarLREGSpKKcoyBJksrMREGSpKKcoyBJksrMREGSpKKcoyBJksrMREGSpKKcoyBJksrMREGSpKJMFCRJUpmZKEiSVFQJznqwoyBJUlElGHpo6o7C9Ed/19FNkJra4k39DpbUGTTzn5no6AZobhExIqU0sqPbITUr3yMlVIKhByczan6M6OgGSE3O94g6nWZOFCRJam4lmKNgoiBJkuoyUdD8cOxVap3vkbJxjoL0CSdpSa3zPaLOyERBkqSinKMgQUTsEBHPRcTYiDi6o9sjNZOIuDAiJkbEkx3dFqkR7CioVRHRFTgb2BEYBuwdEcM6tlVSU7kY2KGjG6EOklLjlw5mR0Ft2RgYm1J6MaU0A7gU2LWD2yQ1jZTSXcC7Hd0OqVGco6C2DAJeq7g9Dtikg9oiSc3FOQpSzUtpd3wWJklaKEwU1JZxwJCK24OBNzqoLZLUXJpgDkGjmSioLQ8Bq0fEyhHRHdgLuLqD2yRJWkjsKKhVKaWZwEHATcAzwGUppac6tlVS84iIS4D7gTUjYlxEHNDRbdJClGY3fulgDj2oTSml64HrO7odUjNKKe3d0W2QGsmOgiRJRc12joIkSSoxEwVJkopqgjkEjWZHQZKkokrQUXDoQZIk1WVHQWqniJgVEY9FxJMRcXlELPFf1LVFRFyb/75La9/KGRF9I+IHBbZxQkQc0d71VWUujoivzMe2VvLbE1VKfimUpArTU0rrpZTWAWYA36u8MzLz/Z5KKV2dUjq1lSJ9gfnuKEjSgmBHQSrmbmC1/Ej6mYg4B/gXMCQitouI+yPiX3ny0AsgInaIiGcj4h7gyy0VRcR+EfG7/PcBETEqIv6dL58DTgVWzdOMX+bljoyIhyLi8Yg4saKuYyPiuYi4FVizrZ2IiO/m9fw7Iv5RlZJsExF3R8R/ImLnvHzXiPhlxbb/9799IqVFWgkuuGRHQZpPEdEN2BF4Il+1JvCnlNL6wPvAccA2KaUNgIeBwyNiceB84EvAF4GBdao/C7gzpfQZYAPgKeBo4IU8zTgyIrYDVif7CvD1gM9GxP9ExGfJLrG9PllHZKN27M6VKaWN8u09A1ReVXAlYHNgJ+D3+T4cAExJKW2U1//diFi5HduRtIjyrAep/XpGxGP573cDFwDLA6+klMbk6zcFhgH3RgRAd7LL+w4FXkopPQ8QEX8BRtTYxlbAtwBSSrOAKRGxVFWZ7fLl0fx2L7KOw5LAqJTSB/k22vOdHOtExElkwxu9yC7V3eKylNJs4PmIeDHfh+2AdSvmL/TJt/2fdmxL6nyaYA5Bo9lRkNpvekppvcoVeWfg/cpVwC3Vl/WNiPVYcF/PHcApKaXzqrZxaIFtXAzsllL6d0TsB2xRcV91XSnf9v9LKVV2KIiIleZzu5IWEQ49SAvWGODzEbEaQEQsERFrAM8CK0fEqnm5et8PMBr4fv7YrhHRG3iPLC1ocRPw7Yq5D4Mioj9wF7B7RPSMiCXJhjnasiTwZkQsBnyj6r49I6JL3uZVgOfybX8/L09ErBERn2rHdqTOqQRzFEwUpAUopfRWfmR+SUT0yFcfl1L6T0SMAK6LiLeBe4B1alRxCDAy/wbCWcD3U0r3R8S9+emHN+TzFNYC7s8TjWnAPimlf0XE34HHgFfIhkfa8mPggbz8E8zdIXkOuBMYAHwvpfRhRPyBbO7CvyLb+FvAbu17diQtiiKVYHxFkqRG6LnpUQ3/EJ0+5hfR6G20xqEHSZJUl0MPkiQV1QRzCBrNREGSJNVloiBJUlElmOdnoiBJkuoyUZAkqSjnKEiSpDIzUZAkqSjnKEiSpDIzUZAkqSjnKEiSpLpSavzShojYISKei4ixEXH0gt5FOwqSJC2iIqIrcDawIzAM2Dsihi3IbTj0IElSUR0/9LAxMDal9CJARFwK7Ao8vaA2YKIgSdKiaxDwWsXtcfm6BcZEQZKkgqY/+ruGfwV0RIwARlSsGplSGtlyd42HLNBzNu0oSJLUxPJOwcg6d48DhlTcHgy8sSC379CDJEmLroeA1SNi5YjoDuwFXL0gN2CiIEnSIiqlNDMiDgJuAroCF6aUnlqQ24hUgstPSpKkYhx6kCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJddlRkCRJdf1/MUCtXFdMnpAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x648 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(9,9))\n",
    "sns.heatmap(cm_morg_val, annot=True, fmt=\".3f\", linewidths=.5, square = True, cmap = 'Blues_r');\n",
    "plt.ylabel('Actual label');\n",
    "plt.xlabel('Predicted label');\n",
    "all_sample_title = 'Accuracy Score: {0}'.format(score_morg_val)\n",
    "plt.title(all_sample_title, size = 15);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Daylight-like fingerprint - logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[781  53]\n",
      " [  0 797]]\n",
      "Accuracy of logistic regression classifier on train set: 0.9967535980954442\n",
      "Accuracy of logistic regression classifier on test set: 0.9661281917665451\n",
      "Accuracy of logistic regression classifier on validation set: 0.967504598405886\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.94      0.97       834\n",
      "           1       0.94      1.00      0.97       797\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      1631\n",
      "   macro avg       0.97      0.97      0.97      1631\n",
      "weighted avg       0.97      0.97      0.97      1631\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#predicting using the test set and creating a confusion matrix\n",
    "predictions_rd_test = lr_rd.predict(x_rd_test)\n",
    "\n",
    "#confustion matrix for test set predictions\n",
    "cm_rd_test = metrics.confusion_matrix(y_rd_test, predictions_rd_test)\n",
    "\n",
    "#predicting using the validation set and creating a confusion matrix\n",
    "predictions_rd_val = lr_rd.predict(x_rd_val)\n",
    "\n",
    "#confustion matrix for test validation predictions\n",
    "cm_rd_val = metrics.confusion_matrix(y_rd_val, predictions_rd_val)\n",
    "\n",
    "print(cm_rd_val)\n",
    "\n",
    "score_rd_train = lr_rd.score(x_rd_train, y_rd_train)\n",
    "print('Accuracy of logistic regression classifier on train set:', score_rd_train)\n",
    "\n",
    "score_rd_test = lr_rd.score(x_rd_test, y_rd_test)\n",
    "print('Accuracy of logistic regression classifier on test set:', score_rd_test)\n",
    "\n",
    "score_rd_val = lr_rd.score(x_rd_val, y_rd_val)\n",
    "print('Accuracy of logistic regression classifier on validation set:', score_rd_val)\n",
    "\n",
    "print(classification_report(y_rd_val, predictions_rd_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Atom-pair fingerprint - logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[856  92]\n",
      " [  0 981]]\n",
      "[[747  83]\n",
      " [  0 810]]\n",
      "Accuracy of logistic regression classifier on train set: 0.993434506511678\n",
      "Accuracy of logistic regression classifier on test set: 0.9523068947641264\n",
      "Accuracy of logistic regression classifier on validation set: 0.949390243902439\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.90      0.95       830\n",
      "           1       0.91      1.00      0.95       810\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      1640\n",
      "   macro avg       0.95      0.95      0.95      1640\n",
      "weighted avg       0.95      0.95      0.95      1640\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#predicting using the test set and creating a confusion matrix\n",
    "predictions_AP_test = lr_AP.predict(x_AP_test)\n",
    "\n",
    "#confustion matrix for test set predictions\n",
    "cm_AP_test = metrics.confusion_matrix(y_AP_test, predictions_AP_test)\n",
    "\n",
    "print(cm_AP_test)\n",
    "\n",
    "#predicting using the validation set and creating a confusion matrix\n",
    "predictions_AP_val = lr_AP.predict(x_AP_val)\n",
    "\n",
    "#confustion matrix for test validation predictions\n",
    "cm_AP_val = metrics.confusion_matrix(y_AP_val, predictions_AP_val)\n",
    "\n",
    "print(cm_AP_val)\n",
    "\n",
    "score_AP_train = lr_AP.score(x_AP_train, y_AP_train)\n",
    "print('Accuracy of logistic regression classifier on train set:', score_AP_train)\n",
    "\n",
    "score_AP_test = lr_AP.score(x_AP_test, y_AP_test)\n",
    "print('Accuracy of logistic regression classifier on test set:', score_AP_test)\n",
    "\n",
    "score_AP_val = lr_AP.score(x_AP_val, y_AP_val)\n",
    "print('Accuracy of logistic regression classifier on validation set:', score_AP_val)\n",
    "\n",
    "print(classification_report(y_AP_val, predictions_AP_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topological torsion fingerprint - logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[830 131]\n",
      " [ 13 944]]\n",
      "[[748  95]\n",
      " [  6 782]]\n",
      "Accuracy of logistic regression classifier on train set: 0.9946952473746887\n",
      "Accuracy of logistic regression classifier on test set: 0.9249217935349322\n",
      "Accuracy of logistic regression classifier on validation set: 0.9380748007357449\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.94       843\n",
      "           1       0.89      0.99      0.94       788\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      1631\n",
      "   macro avg       0.94      0.94      0.94      1631\n",
      "weighted avg       0.94      0.94      0.94      1631\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#predicting using the test set and creating a confusion matrix\n",
    "predictions_torsion_test = lr_torsion.predict(x_torsion_test)\n",
    "\n",
    "#confustion matrix for test set predictions\n",
    "cm_torsion_test = metrics.confusion_matrix(y_torsion_test, predictions_torsion_test)\n",
    "\n",
    "print(cm_torsion_test)\n",
    "\n",
    "#predicting using the validation set and creating a confusion matrix\n",
    "predictions_torsion_val = lr_torsion.predict(x_torsion_val)\n",
    "\n",
    "#confustion matrix for test validation predictions\n",
    "cm_torsion_val = metrics.confusion_matrix(y_torsion_val, predictions_torsion_val)\n",
    "\n",
    "print(cm_torsion_val)\n",
    "\n",
    "score_torsion_train = lr_torsion.score(x_torsion_train, y_torsion_train)\n",
    "print('Accuracy of logistic regression classifier on train set:', score_torsion_train)\n",
    "\n",
    "score_torsion_test = lr_torsion.score(x_torsion_test, y_torsion_test)\n",
    "print('Accuracy of logistic regression classifier on test set:', score_torsion_test)\n",
    "\n",
    "score_torsion_val = lr_torsion.score(x_torsion_val, y_torsion_val)\n",
    "print('Accuracy of logistic regression classifier on validation set:', score_torsion_val)\n",
    "\n",
    "print(classification_report(y_torsion_val, predictions_torsion_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# k-nearest neighbor\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid searching and model fitting\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8661400281354832"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "grid_params = {'n_neighbors': [3, 5, 7, 9, 11, 13, 15, 17, 19], 'weights': ['distance'], 'metric': ['euclidean']}\n",
    "knn_cv = GridSearchCV(knn, grid_params, cv=2, verbose = 1, n_jobs = -1)\n",
    "\n",
    "knn_morg = knn_cv.fit(x_morg_train, y_morg_train)\n",
    "\n",
    "knn_cv.best_params_\n",
    "knn_cv.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8460342146189735\n",
      "0.8460342146189735\n",
      "0.8154484188698807\n",
      "0.7884914463452566\n"
     ]
    }
   ],
   "source": [
    "knn_3n = KNeighborsClassifier(n_neighbors=3)\n",
    "knn_AP = knn3.fit(x_AP_train, y_AP_train)\n",
    "\n",
    "knn_5n = KNeighborsClassifier(n_neighbors=5)\n",
    "knn_AP5 = knn_5n.fit(x_AP_train, y_AP_train)\n",
    "\n",
    "knn_7n = KNeighborsClassifier(n_neighbors=7)\n",
    "knn_AP7 = knn_7n.fit(x_AP_train, y_AP_train)\n",
    "\n",
    "knn_9n = KNeighborsClassifier(n_neighbors=9)\n",
    "knn_AP9 = knn_9n.fit(x_AP_train, y_AP_train)\n",
    "\n",
    "knn_AP_3n = knn_AP.score(x_AP_test, y_AP_test)\n",
    "print(knn_AP_3n)\n",
    "knn_AP_5n = knn_AP5.score(x_AP_test, y_AP_test)\n",
    "print(knn_AP_5n)\n",
    "knn_AP_7n = knn_AP7.score(x_AP_test, y_AP_test)\n",
    "print(knn_AP_7n)\n",
    "knn_AP_9n = knn_AP9.score(x_AP_test, y_AP_test)\n",
    "print(knn_AP_9n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the models and writing a function for scoring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn3 = KNeighborsClassifier(n_neighbors=3)\n",
    "\n",
    "knn_morg = knn3.fit(x_morg_train, y_morg_train)\n",
    "knn_rd = knn3.fit(x_rd_train, y_rd_train)\n",
    "knn_AP = knn3.fit(x_AP_train, y_AP_train)\n",
    "knn_torsion = knn3.fit(x_torsion_train, y_torsion_train)\n",
    "\n",
    "def mod_acc_knn(x_train, y_train, x_test, y_test):\n",
    "    knn3 = KNeighborsClassifier(n_neighbors=3)\n",
    "    knn_fp = knn3.fit(x_train, y_train)\n",
    "    score_fp_test_knn = knn_fp.score(x_test, y_test)\n",
    "    return score_fp_test_knn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### knn training data accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9549773267112934\n",
      "0.9519532518125744\n",
      "0.9316542890969756\n",
      "0.9172891631482083\n"
     ]
    }
   ],
   "source": [
    "knn_morg_acc_train = mod_acc_knn(x_morg_train, y_morg_train, x_morg_train, y_morg_train)\n",
    "print(knn_morg_acc_train)\n",
    "knn_rd_acc_train = mod_acc_knn(x_rd_train, y_rd_train, x_rd_train, y_rd_train)\n",
    "print(knn_rd_acc_train)\n",
    "knn_AP_acc_train = mod_acc_knn(x_AP_train, y_AP_train, x_AP_train, y_AP_train)\n",
    "print(knn_AP_acc_train)\n",
    "knn_torsion_acc_train = mod_acc_knn(x_torsion_train, y_torsion_train, x_torsion_train, y_torsion_train)\n",
    "print(knn_torsion_acc_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### knn test data accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9088066701406983\n",
      "0.8750648004147227\n",
      "0.8607924921793535\n"
     ]
    }
   ],
   "source": [
    "knn_morg_acc_test = mod_acc_knn(x_morg_train, y_morg_train, x_morg_test, y_morg_test)\n",
    "print(knn_morg_acc_test)\n",
    "knn_rd_acc_test = mod_acc_knn(x_rd_train, y_rd_train, x_rd_test, y_rd_test)\n",
    "print(knn_rd_acc_test)\n",
    "knn_AP_acc_test = mod_acc_knn(x_AP_train, y_AP_train, x_AP_test, y_AP_test)\n",
    "print(knn_AP_acc_test)\n",
    "knn_torsion_acc_test = mod_acc_knn(x_torsion_train, y_torsion_train, x_torsion_test, y_torsion_test)\n",
    "print(knn_torsion_acc_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### knn validation data accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.908868501529052\n",
      "0.9141630901287554\n",
      "0.8640243902439024\n",
      "0.8553034947884733\n"
     ]
    }
   ],
   "source": [
    "knn_morg_acc_val = mod_acc_knn(x_morg_train, y_morg_train, x_morg_val, y_morg_val)\n",
    "print(knn_morg_acc_val)\n",
    "knn_rd_acc_val = mod_acc_knn(x_rd_train, y_rd_train, x_rd_val, y_rd_val)\n",
    "print(knn_rd_acc_val)\n",
    "knn_AP_acc_val = mod_acc_knn(x_AP_train, y_AP_train, x_AP_val, y_AP_val)\n",
    "print(knn_AP_acc_val)\n",
    "knn_torsion_acc_val = mod_acc_knn(x_torsion_train, y_torsion_train, x_torsion_val, y_torsion_val)\n",
    "print(knn_torsion_acc_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient boosting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid searching and model training, and metrics\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set search parameters\n",
    "params_gb = {\n",
    "    \"loss\":[\"deviance\"],\n",
    "    \"learning_rate\": [1.0],\n",
    "    \"min_samples_split\": np.linspace(0.1, 0.5, 12),\n",
    "    \"min_samples_leaf\": np.linspace(0.1, 0.5, 12),\n",
    "    \"max_depth\":[3,5,7],\n",
    "    \"max_features\":[\"sqrt\"],\n",
    "    \"criterion\": [\"friedman_mse\"],\n",
    "    \"subsample\":[0.5, 0.75, 0.95],\n",
    "    \"n_estimators\":[100]\n",
    "    }\n",
    "\n",
    "#initialize grid seach\n",
    "gb = GridSearchCV(GradientBoostingClassifier(), params_gb, cv=2, n_jobs=-1, verbose = 2)\n",
    "#train the model while tuning \n",
    "gb_morg = gb.fit(x_morg_train, y_morg_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check out the accuracy on the test and validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9069315482617145\n",
      "0.9011960478419136\n",
      "0.9113149847094801\n"
     ]
    }
   ],
   "source": [
    "print(gb_morg.score(x_morg_train, y_morg_train))\n",
    "print(gb_morg.score(x_morg_test, y_morg_test))\n",
    "print(gb_morg.score(x_morg_val, y_morg_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#initialize grid seach\n",
    "gb = GridSearchCV(GradientBoostingClassifier(), params_gb, cv=2, n_jobs=-1, verbose = 2)\n",
    "#train the model while tuning \n",
    "gb_rd = gb.fit(x_rd_train, y_rd_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9853911914294989\n",
      "0.9614382490880667\n",
      "0.9601471489883507\n"
     ]
    }
   ],
   "source": [
    "print(gb_rd.score(x_rd_train, y_rd_train))\n",
    "print(gb_rd.score(x_rd_test, y_rd_test))\n",
    "print(gb_rd.score(x_rd_val, y_rd_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#initialize grid seach\n",
    "gb = GridSearchCV(GradientBoostingClassifier(), params_gb, cv=2, n_jobs=-1, verbose = 2)\n",
    "#train the model while tuning \n",
    "gb_AP = gb.fit(x_AP_train, y_AP_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9698633085781939\n",
      "0.9548989113530326\n",
      "0.9640243902439024\n"
     ]
    }
   ],
   "source": [
    "print(gb_AP.score(x_AP_train, y_AP_train))\n",
    "print(gb_AP.score(x_AP_test, y_AP_test))\n",
    "print(gb_AP.score(x_AP_val, y_AP_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#initialize grid seach\n",
    "gb = GridSearchCV(GradientBoostingClassifier(), params_gb, cv=2, n_jobs=-1, verbose = 2)\n",
    "#train the model while tuning \n",
    "gb_torsion = gb.fit(x_torsion_train, y_torsion_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.866623362563603\n",
      "0.8576642335766423\n",
      "0.8583690987124464\n"
     ]
    }
   ],
   "source": [
    "print(gb_torsion_3.score(x_torsion_train, y_torsion_train))\n",
    "print(gb_torsion_3.score(x_torsion_test, y_torsion_test))\n",
    "print(gb_torsion_3.score(x_torsion_val, y_torsion_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUC values and F1-scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AUC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auc_calc(x_val, y_val, model):\n",
    "    # Probability predictions using validation data\n",
    "    prob = model.predict_proba(x_val)\n",
    "    # Correct probability predictions\n",
    "    prob = prob[:, 1]\n",
    "    # Calculate area under the curve with validation data labels and correct predicted probabilities\n",
    "    auc = metrics.roc_auc_score(y_val, prob)\n",
    "    # Return the AUC score with 6 figures\n",
    "    return 'AUC: %.6f' % auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Morgan fingerprint auc values for all three models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC using logistic regression:  AUC: 0.973971\n",
      "AUC using k-nearest naighbor:  AUC: 0.957206\n",
      "AUC using gradient boosting:  AUC: 0.965234\n"
     ]
    }
   ],
   "source": [
    "morg_lr_auc = auc_calc(x_morg_val, y_morg_val, lr_morg)\n",
    "print(\"AUC using logistic regression: \", morg_lr_auc)\n",
    "morg_knn_auc = auc_calc(x_morg_val, y_morg_val, knn_morg)\n",
    "print(\"AUC using k-nearest naighbor: \", morg_knn_auc)\n",
    "morg_gb_auc = auc_calc(x_morg_val, y_morg_val, gb_morg)\n",
    "print(\"AUC using gradient boosting: \", morg_gb_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Daylight-like fingerprint auc values for all three models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC using logistic regression:  AUC: 0.980311\n",
      "AUC using k-nearest naighbor:  AUC: 0.957979\n",
      "AUC using gradient boosting:  AUC: 0.986917\n"
     ]
    }
   ],
   "source": [
    "rd_lr_auc = auc_calc(x_rd_val, y_rd_val, lr_rd)\n",
    "print(\"AUC using logistic regression: \", rd_lr_auc)\n",
    "rd_knn_auc = auc_calc(x_rd_val, y_rd_val, knn_rd)\n",
    "print(\"AUC using k-nearest neighbor: \", rd_knn_auc)\n",
    "rd_gb_auc = auc_calc(x_rd_val, y_rd_val, gb_rd)\n",
    "print(\"AUC using gradient boosting: \", rd_gb_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Atom-pair fingerprint auc values for all three models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC using logistic regression:  AUC: 0.963440\n",
      "AUC using k-nearest naighbor:  AUC: 0.584849\n",
      "AUC using gradient boosting:  AUC: 0.988373\n"
     ]
    }
   ],
   "source": [
    "AP_lr_auc = auc_calc(x_AP_val, y_AP_val, lr_AP)\n",
    "print(\"AUC using logistic regression: \", AP_lr_auc)\n",
    "AP_knn_auc = auc_calc(x_AP_val, y_AP_val, knn_AP)\n",
    "print(\"AUC using k-nearest naighbor: \", AP_knn_auc)\n",
    "AP_gb_auc = auc_calc(x_AP_val, y_AP_val, gb_AP)\n",
    "print(\"AUC using gradient boosting: \", AP_gb_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Topologial torsion fingerprint auc values for all three models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC using logistic regression:  AUC: 0.930011\n",
      "AUC using k-nearest naighbor:  AUC: 0.931734\n",
      "AUC using gradient boosting:  AUC: 0.716465\n"
     ]
    }
   ],
   "source": [
    "torsion_lr_auc = auc_calc(x_torsion_val, y_torsion_val, lr_torsion)\n",
    "print(\"AUC using logistic regression: \", torsion_lr_auc)\n",
    "torsion_knn_auc = auc_calc(x_torsion_val, y_torsion_val, knn_torsion)\n",
    "print(\"AUC using k-nearest naighbor: \", torsion_knn_auc)\n",
    "torsion_gb_auc = auc_calc(x_torsion_val, y_torsion_val, gb_torsion)\n",
    "print(\"AUC using gradient boosting: \", torsion_gb_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## f1-score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1score(x_val, y_val, model):\n",
    "    #predictions made on validation data\n",
    "    predictions = model.predict(x_val)\n",
    "    #generating the classification report\n",
    "    clrpt = classification_report(y_val, predictions, output_dict = True)\n",
    "    #accessing the weighted average entery in dictionairy\n",
    "    d = clrpt[\"weighted avg\"]\n",
    "    #calling the f1_score\n",
    "    f1 = d['f1-score']\n",
    "    return 'f1-score %.6f' % f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Morgan fingerprint f-scores for all three models using validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1-score 0.952264\n",
      "f1-score 0.402606\n",
      "f1-score 0.911232\n"
     ]
    }
   ],
   "source": [
    "morg_lr_f1 = f1score(x_morg_val, y_morg_val, lr_morg)\n",
    "morg_knn_f1 = f1score(x_morg_val, y_morg_val, knn_morg)\n",
    "morg_gb_f1 = f1score(x_morg_val, y_morg_val, gb_morg)\n",
    "print(morg_lr_f1)\n",
    "print(morg_knn_f1)\n",
    "print(morg_gb_f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Daylight-like fingerprint f-scores for all three models using validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1-score 0.967494\n",
      "f1-score 0.469449\n",
      "f1-score 0.960150\n"
     ]
    }
   ],
   "source": [
    "rd_lr_f1 = f1score(x_rd_val, y_rd_val, lr_rd)\n",
    "rd_knn_f1 = f1score(x_rd_val, y_rd_val, knn_rd)\n",
    "rd_gb_f1 = f1score(x_rd_val, y_rd_val, gb_rd)\n",
    "print(rd_lr_f1)\n",
    "print(rd_knn_f1)\n",
    "print(rd_gb_f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Atom-pair fingerprint f-scores for all three models using validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1-score 0.949292\n",
      "f1-score 0.569587\n",
      "f1-score 0.964024\n"
     ]
    }
   ],
   "source": [
    "AP_lr_f1 = f1score(x_AP_val, y_AP_val, lr_AP)\n",
    "AP_knn_f1 = f1score(x_AP_val, y_AP_val, knn_AP)\n",
    "AP_gb_f1 = f1score(x_AP_val, y_AP_val, gb_AP)\n",
    "print(AP_lr_f1)\n",
    "print(AP_knn_f1)\n",
    "print(AP_gb_f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Topological torsion fingerprint f-scores for all three models using validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1-score 0.938004\n",
      "f1-score 0.852951\n",
      "f1-score 0.651235\n"
     ]
    }
   ],
   "source": [
    "torsion_lr_f1 = f1score(x_torsion_val, y_torsion_val, lr_torsion)\n",
    "torsion_knn_f1 = f1score(x_torsion_val, y_torsion_val, knn_torsion)\n",
    "torsion_gb_f1 = f1score(x_torsion_val, y_torsion_val, gb_torsion)\n",
    "print(torsion_lr_f1)\n",
    "print(torsion_knn_f1)\n",
    "print(torsion_gb_f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results and conclusion\n",
    "\n",
    "\n",
    "\n",
    "<img src=\"images/conclusion_table.png\">\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}