Softmax.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "author=\"Jumabek Alikhanov\"\n",
    "date = 'Nov 19,2019'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports\n",
    "import os\n",
    "from os.path import join\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "#Please download dataset from http://205.174.165.80/CICDataset/CIC-IDS-2017/\n",
    "dataroot = 'MachineLearningCVE/'\n",
    "SEED=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MachineLearningCVE/*.pcap_ISCX.csv\n",
      "there are 2830743 flow records with 79 feature dimension\n",
      "stripped column names\n",
      "dropped bad columns\n",
      "There are 0 nan entries\n",
      "converted to numeric\n"
     ]
    }
   ],
   "source": [
    "%autoreload 2\n",
    "from preprocessing import load_data\n",
    "X,y = load_data(dataroot) # reads csv file and returns np array of X,y -> of shape (N,D) and (N,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Imbalance\n",
    "1. It is curucial to adress this issue in order to get decent performance\n",
    "2. It also affects evaluation, we should calculate  `balanced accuracy`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocessing import balance_data, normalize\n",
    "X = normalize(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "%autoreload 2\n",
    "from models import Classifier\n",
    "\n",
    "def ensure_dir(dir_path):\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "\n",
    "def getClassifier(args,runs_dir=None):\n",
    "    \n",
    "    (method,optim,lr,reg,batch_size,input_dim,num_class,num_epochs) = args\n",
    "    if runs_dir is not None:\n",
    "        ensure_dir(runs_dir)\n",
    "    \n",
    "    clf = Classifier(method,input_dim,num_class,lr=lr,reg=reg,num_epochs=num_epochs,\n",
    "                        batch_size=batch_size,runs_dir=runs_dir)\n",
    "    return clf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------\n",
      "Fold #0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2264586, 76)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn import metrics\n",
    "K=5\n",
    "skf = StratifiedKFold(n_splits=K,random_state=SEED)\n",
    "for fold_index, (train_index,test_index) in enumerate(skf.split(X,y)):# runs only once \n",
    "        print('---------------------------------------------')\n",
    "        print('Fold #{}'.format(fold_index))    \n",
    "        X_train = X[train_index]\n",
    "        y_train = y[train_index]\n",
    "        X_test = X[test_index]\n",
    "        y_test = y[test_index]\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 2.7035\n",
      "Epoch [1/20], Step [100/398], Loss: 2.6841\n",
      "Epoch [1/20], Step [150/398], Loss: 2.6640\n",
      "Epoch [1/20], Step [200/398], Loss: 2.6429\n",
      "Epoch [1/20], Step [250/398], Loss: 2.6241\n",
      "Epoch [1/20], Step [300/398], Loss: 2.6030\n",
      "Epoch [1/20], Step [350/398], Loss: 2.5855\n",
      "Epoch [2/20], Step [1/398], Loss: 2.5660\n",
      "Epoch [2/20], Step [51/398], Loss: 2.5475\n",
      "Epoch [2/20], Step [101/398], Loss: 2.5262\n",
      "Epoch [2/20], Step [151/398], Loss: 2.5121\n",
      "Epoch [2/20], Step [201/398], Loss: 2.4996\n",
      "Epoch [2/20], Step [251/398], Loss: 2.4747\n",
      "Epoch [2/20], Step [301/398], Loss: 2.4643\n",
      "Epoch [2/20], Step [351/398], Loss: 2.4435\n",
      "Epoch [3/20], Step [2/398], Loss: 2.4225\n",
      "Epoch [3/20], Step [52/398], Loss: 2.4107\n",
      "Epoch [3/20], Step [102/398], Loss: 2.3868\n",
      "Epoch [3/20], Step [152/398], Loss: 2.3747\n",
      "Epoch [3/20], Step [202/398], Loss: 2.3706\n",
      "Epoch [3/20], Step [252/398], Loss: 2.3569\n",
      "Epoch [3/20], Step [302/398], Loss: 2.3387\n",
      "Epoch [3/20], Step [352/398], Loss: 2.3201\n",
      "Epoch [4/20], Step [3/398], Loss: 2.3097\n",
      "Epoch [4/20], Step [53/398], Loss: 2.2948\n",
      "Epoch [4/20], Step [103/398], Loss: 2.2761\n",
      "Epoch [4/20], Step [153/398], Loss: 2.2533\n",
      "Epoch [4/20], Step [203/398], Loss: 2.2425\n",
      "Epoch [4/20], Step [253/398], Loss: 2.2347\n",
      "Epoch [4/20], Step [303/398], Loss: 2.2201\n",
      "Epoch [4/20], Step [353/398], Loss: 2.1978\n",
      "Epoch [5/20], Step [4/398], Loss: 2.1875\n",
      "Epoch [5/20], Step [54/398], Loss: 2.1785\n",
      "Epoch [5/20], Step [104/398], Loss: 2.1651\n",
      "Epoch [5/20], Step [154/398], Loss: 2.1619\n",
      "Epoch [5/20], Step [204/398], Loss: 2.1365\n",
      "Epoch [5/20], Step [254/398], Loss: 2.1317\n",
      "Epoch [5/20], Step [304/398], Loss: 2.1168\n",
      "Epoch [5/20], Step [354/398], Loss: 2.1033\n",
      "Epoch [6/20], Step [5/398], Loss: 2.0860\n",
      "Epoch [6/20], Step [55/398], Loss: 2.0860\n",
      "Epoch [6/20], Step [105/398], Loss: 2.0742\n",
      "Epoch [6/20], Step [155/398], Loss: 2.0464\n",
      "Epoch [6/20], Step [205/398], Loss: 2.0392\n",
      "Epoch [6/20], Step [255/398], Loss: 2.0509\n",
      "Epoch [6/20], Step [305/398], Loss: 2.0365\n",
      "Epoch [6/20], Step [355/398], Loss: 2.0120\n",
      "Epoch [7/20], Step [6/398], Loss: 1.9916\n",
      "Epoch [7/20], Step [56/398], Loss: 2.0128\n",
      "Epoch [7/20], Step [106/398], Loss: 1.9970\n",
      "Epoch [7/20], Step [156/398], Loss: 1.9827\n",
      "Epoch [7/20], Step [206/398], Loss: 1.9558\n",
      "Epoch [7/20], Step [256/398], Loss: 1.9610\n",
      "Epoch [7/20], Step [306/398], Loss: 1.9658\n",
      "Epoch [7/20], Step [356/398], Loss: 1.9481\n",
      "Epoch [8/20], Step [7/398], Loss: 1.9322\n",
      "Epoch [8/20], Step [57/398], Loss: 1.9209\n",
      "Epoch [8/20], Step [107/398], Loss: 1.9120\n",
      "Epoch [8/20], Step [157/398], Loss: 1.8966\n",
      "Epoch [8/20], Step [207/398], Loss: 1.9111\n",
      "Epoch [8/20], Step [257/398], Loss: 1.8772\n",
      "Epoch [8/20], Step [307/398], Loss: 1.8939\n",
      "Epoch [8/20], Step [357/398], Loss: 1.8674\n",
      "Epoch [9/20], Step [8/398], Loss: 1.8697\n",
      "Epoch [9/20], Step [58/398], Loss: 1.8543\n",
      "Epoch [9/20], Step [108/398], Loss: 1.8480\n",
      "Epoch [9/20], Step [158/398], Loss: 1.8415\n",
      "Epoch [9/20], Step [208/398], Loss: 1.8413\n",
      "Epoch [9/20], Step [258/398], Loss: 1.8182\n",
      "Epoch [9/20], Step [308/398], Loss: 1.8144\n",
      "Epoch [9/20], Step [358/398], Loss: 1.7887\n",
      "Epoch [10/20], Step [9/398], Loss: 1.7884\n",
      "Epoch [10/20], Step [59/398], Loss: 1.8018\n",
      "Epoch [10/20], Step [109/398], Loss: 1.7925\n",
      "Epoch [10/20], Step [159/398], Loss: 1.7663\n",
      "Epoch [10/20], Step [209/398], Loss: 1.7764\n",
      "Epoch [10/20], Step [259/398], Loss: 1.7884\n",
      "Epoch [10/20], Step [309/398], Loss: 1.7745\n",
      "Epoch [10/20], Step [359/398], Loss: 1.7415\n",
      "Epoch [11/20], Step [10/398], Loss: 1.7494\n",
      "Epoch [11/20], Step [60/398], Loss: 1.7324\n",
      "Epoch [11/20], Step [110/398], Loss: 1.7288\n",
      "Epoch [11/20], Step [160/398], Loss: 1.7217\n",
      "Epoch [11/20], Step [210/398], Loss: 1.7175\n",
      "Epoch [11/20], Step [260/398], Loss: 1.7069\n",
      "Epoch [11/20], Step [310/398], Loss: 1.7029\n",
      "Epoch [11/20], Step [360/398], Loss: 1.6888\n",
      "Epoch [12/20], Step [11/398], Loss: 1.7023\n",
      "Epoch [12/20], Step [61/398], Loss: 1.6826\n",
      "Epoch [12/20], Step [111/398], Loss: 1.6918\n",
      "Epoch [12/20], Step [161/398], Loss: 1.6586\n",
      "Epoch [12/20], Step [211/398], Loss: 1.6733\n",
      "Epoch [12/20], Step [261/398], Loss: 1.6463\n",
      "Epoch [12/20], Step [311/398], Loss: 1.6574\n",
      "Epoch [12/20], Step [361/398], Loss: 1.6270\n",
      "Epoch [13/20], Step [12/398], Loss: 1.6499\n",
      "Epoch [13/20], Step [62/398], Loss: 1.6405\n",
      "Epoch [13/20], Step [112/398], Loss: 1.6415\n",
      "Epoch [13/20], Step [162/398], Loss: 1.6228\n",
      "Epoch [13/20], Step [212/398], Loss: 1.6286\n",
      "Epoch [13/20], Step [262/398], Loss: 1.6215\n",
      "Epoch [13/20], Step [312/398], Loss: 1.6447\n",
      "Epoch [13/20], Step [362/398], Loss: 1.6159\n",
      "Epoch [14/20], Step [13/398], Loss: 1.6016\n",
      "Epoch [14/20], Step [63/398], Loss: 1.5920\n",
      "Epoch [14/20], Step [113/398], Loss: 1.6011\n",
      "Epoch [14/20], Step [163/398], Loss: 1.5869\n",
      "Epoch [14/20], Step [213/398], Loss: 1.5720\n",
      "Epoch [14/20], Step [263/398], Loss: 1.5717\n",
      "Epoch [14/20], Step [313/398], Loss: 1.5815\n",
      "Epoch [14/20], Step [363/398], Loss: 1.5605\n",
      "Epoch [15/20], Step [14/398], Loss: 1.5731\n",
      "Epoch [15/20], Step [64/398], Loss: 1.5551\n",
      "Epoch [15/20], Step [114/398], Loss: 1.5495\n",
      "Epoch [15/20], Step [164/398], Loss: 1.5551\n",
      "Epoch [15/20], Step [214/398], Loss: 1.5257\n",
      "Epoch [15/20], Step [264/398], Loss: 1.5452\n",
      "Epoch [15/20], Step [314/398], Loss: 1.5283\n",
      "Epoch [15/20], Step [364/398], Loss: 1.5444\n",
      "Epoch [16/20], Step [15/398], Loss: 1.5369\n",
      "Epoch [16/20], Step [65/398], Loss: 1.5133\n",
      "Epoch [16/20], Step [115/398], Loss: 1.5226\n",
      "Epoch [16/20], Step [165/398], Loss: 1.5147\n",
      "Epoch [16/20], Step [215/398], Loss: 1.5260\n",
      "Epoch [16/20], Step [265/398], Loss: 1.5026\n",
      "Epoch [16/20], Step [315/398], Loss: 1.5096\n",
      "Epoch [16/20], Step [365/398], Loss: 1.4975\n",
      "Epoch [17/20], Step [16/398], Loss: 1.4965\n",
      "Epoch [17/20], Step [66/398], Loss: 1.5026\n",
      "Epoch [17/20], Step [116/398], Loss: 1.4762\n",
      "Epoch [17/20], Step [166/398], Loss: 1.4871\n",
      "Epoch [17/20], Step [216/398], Loss: 1.4948\n",
      "Epoch [17/20], Step [266/398], Loss: 1.4777\n",
      "Epoch [17/20], Step [316/398], Loss: 1.4743\n",
      "Epoch [17/20], Step [366/398], Loss: 1.4628\n",
      "Epoch [18/20], Step [17/398], Loss: 1.4534\n",
      "Epoch [18/20], Step [67/398], Loss: 1.4547\n",
      "Epoch [18/20], Step [117/398], Loss: 1.4493\n",
      "Epoch [18/20], Step [167/398], Loss: 1.4536\n",
      "Epoch [18/20], Step [217/398], Loss: 1.4556\n",
      "Epoch [18/20], Step [267/398], Loss: 1.4597\n",
      "Epoch [18/20], Step [317/398], Loss: 1.4449\n",
      "Epoch [18/20], Step [367/398], Loss: 1.4034\n",
      "Epoch [19/20], Step [18/398], Loss: 1.4380\n",
      "Epoch [19/20], Step [68/398], Loss: 1.4210\n",
      "Epoch [19/20], Step [118/398], Loss: 1.4145\n",
      "Epoch [19/20], Step [168/398], Loss: 1.4134\n",
      "Epoch [19/20], Step [218/398], Loss: 1.4360\n",
      "Epoch [19/20], Step [268/398], Loss: 1.4406\n",
      "Epoch [19/20], Step [318/398], Loss: 1.4003\n",
      "Epoch [19/20], Step [368/398], Loss: 1.3982\n",
      "Epoch [20/20], Step [19/398], Loss: 1.3901\n",
      "Epoch [20/20], Step [69/398], Loss: 1.4074\n",
      "Epoch [20/20], Step [119/398], Loss: 1.3972\n",
      "Epoch [20/20], Step [169/398], Loss: 1.4028\n",
      "Epoch [20/20], Step [219/398], Loss: 1.4066\n",
      "Epoch [20/20], Step [269/398], Loss: 1.3955\n",
      "Epoch [20/20], Step [319/398], Loss: 1.3934\n",
      "Epoch [20/20], Step [369/398], Loss: 1.3686\n",
      "Model is trained in 737 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 2.6707\n",
      "Epoch [1/20], Step [100/398], Loss: 2.6525\n",
      "Epoch [1/20], Step [150/398], Loss: 2.6325\n",
      "Epoch [1/20], Step [200/398], Loss: 2.6110\n",
      "Epoch [1/20], Step [250/398], Loss: 2.5945\n",
      "Epoch [1/20], Step [300/398], Loss: 2.5740\n",
      "Epoch [1/20], Step [350/398], Loss: 2.5521\n",
      "Epoch [2/20], Step [1/398], Loss: 2.5358\n",
      "Epoch [2/20], Step [51/398], Loss: 2.5145\n",
      "Epoch [2/20], Step [101/398], Loss: 2.4992\n",
      "Epoch [2/20], Step [151/398], Loss: 2.4848\n",
      "Epoch [2/20], Step [201/398], Loss: 2.4730\n",
      "Epoch [2/20], Step [251/398], Loss: 2.4489\n",
      "Epoch [2/20], Step [301/398], Loss: 2.4386\n",
      "Epoch [2/20], Step [351/398], Loss: 2.4208\n",
      "Epoch [3/20], Step [2/398], Loss: 2.3994\n",
      "Epoch [3/20], Step [52/398], Loss: 2.3905\n",
      "Epoch [3/20], Step [102/398], Loss: 2.3768\n",
      "Epoch [3/20], Step [152/398], Loss: 2.3542\n",
      "Epoch [3/20], Step [202/398], Loss: 2.3465\n",
      "Epoch [3/20], Step [252/398], Loss: 2.3310\n",
      "Epoch [3/20], Step [302/398], Loss: 2.3199\n",
      "Epoch [3/20], Step [352/398], Loss: 2.2915\n",
      "Epoch [4/20], Step [3/398], Loss: 2.2876\n",
      "Epoch [4/20], Step [53/398], Loss: 2.2611\n",
      "Epoch [4/20], Step [103/398], Loss: 2.2589\n",
      "Epoch [4/20], Step [153/398], Loss: 2.2457\n",
      "Epoch [4/20], Step [203/398], Loss: 2.2267\n",
      "Epoch [4/20], Step [253/398], Loss: 2.2198\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [4/20], Step [303/398], Loss: 2.2057\n",
      "Epoch [4/20], Step [353/398], Loss: 2.1714\n",
      "Epoch [5/20], Step [4/398], Loss: 2.1746\n",
      "Epoch [5/20], Step [54/398], Loss: 2.1562\n",
      "Epoch [5/20], Step [104/398], Loss: 2.1477\n",
      "Epoch [5/20], Step [154/398], Loss: 2.1573\n",
      "Epoch [5/20], Step [204/398], Loss: 2.1141\n",
      "Epoch [5/20], Step [254/398], Loss: 2.1221\n",
      "Epoch [5/20], Step [304/398], Loss: 2.1033\n",
      "Epoch [5/20], Step [354/398], Loss: 2.0927\n",
      "Epoch [6/20], Step [5/398], Loss: 2.0764\n",
      "Epoch [6/20], Step [55/398], Loss: 2.0745\n",
      "Epoch [6/20], Step [105/398], Loss: 2.0593\n",
      "Epoch [6/20], Step [155/398], Loss: 2.0408\n",
      "Epoch [6/20], Step [205/398], Loss: 2.0246\n",
      "Epoch [6/20], Step [255/398], Loss: 2.0168\n",
      "Epoch [6/20], Step [305/398], Loss: 2.0311\n",
      "Epoch [6/20], Step [355/398], Loss: 2.0044\n",
      "Epoch [7/20], Step [6/398], Loss: 1.9947\n",
      "Epoch [7/20], Step [56/398], Loss: 1.9770\n",
      "Epoch [7/20], Step [106/398], Loss: 1.9763\n",
      "Epoch [7/20], Step [156/398], Loss: 1.9729\n",
      "Epoch [7/20], Step [206/398], Loss: 1.9476\n",
      "Epoch [7/20], Step [256/398], Loss: 1.9537\n",
      "Epoch [7/20], Step [306/398], Loss: 1.9607\n",
      "Epoch [7/20], Step [356/398], Loss: 1.9319\n",
      "Epoch [8/20], Step [7/398], Loss: 1.9158\n",
      "Epoch [8/20], Step [57/398], Loss: 1.9099\n",
      "Epoch [8/20], Step [107/398], Loss: 1.9127\n",
      "Epoch [8/20], Step [157/398], Loss: 1.9113\n",
      "Epoch [8/20], Step [207/398], Loss: 1.8978\n",
      "Epoch [8/20], Step [257/398], Loss: 1.8700\n",
      "Epoch [8/20], Step [307/398], Loss: 1.8736\n",
      "Epoch [8/20], Step [357/398], Loss: 1.8430\n",
      "Epoch [9/20], Step [8/398], Loss: 1.8632\n",
      "Epoch [9/20], Step [58/398], Loss: 1.8463\n",
      "Epoch [9/20], Step [108/398], Loss: 1.8411\n",
      "Epoch [9/20], Step [158/398], Loss: 1.8323\n",
      "Epoch [9/20], Step [208/398], Loss: 1.8287\n",
      "Epoch [9/20], Step [258/398], Loss: 1.8131\n",
      "Epoch [9/20], Step [308/398], Loss: 1.7979\n",
      "Epoch [9/20], Step [358/398], Loss: 1.7806\n",
      "Epoch [10/20], Step [9/398], Loss: 1.7858\n",
      "Epoch [10/20], Step [59/398], Loss: 1.7878\n",
      "Epoch [10/20], Step [109/398], Loss: 1.7846\n",
      "Epoch [10/20], Step [159/398], Loss: 1.7538\n",
      "Epoch [10/20], Step [209/398], Loss: 1.7745\n",
      "Epoch [10/20], Step [259/398], Loss: 1.7466\n",
      "Epoch [10/20], Step [309/398], Loss: 1.7557\n",
      "Epoch [10/20], Step [359/398], Loss: 1.7146\n",
      "Epoch [11/20], Step [10/398], Loss: 1.7486\n",
      "Epoch [11/20], Step [60/398], Loss: 1.7311\n",
      "Epoch [11/20], Step [110/398], Loss: 1.7306\n",
      "Epoch [11/20], Step [160/398], Loss: 1.7143\n",
      "Epoch [11/20], Step [210/398], Loss: 1.6993\n",
      "Epoch [11/20], Step [260/398], Loss: 1.7106\n",
      "Epoch [11/20], Step [310/398], Loss: 1.6928\n",
      "Epoch [11/20], Step [360/398], Loss: 1.6750\n",
      "Epoch [12/20], Step [11/398], Loss: 1.6730\n",
      "Epoch [12/20], Step [61/398], Loss: 1.6714\n",
      "Epoch [12/20], Step [111/398], Loss: 1.6683\n",
      "Epoch [12/20], Step [161/398], Loss: 1.6693\n",
      "Epoch [12/20], Step [211/398], Loss: 1.6670\n",
      "Epoch [12/20], Step [261/398], Loss: 1.6436\n",
      "Epoch [12/20], Step [311/398], Loss: 1.6450\n",
      "Epoch [12/20], Step [361/398], Loss: 1.6313\n",
      "Epoch [13/20], Step [12/398], Loss: 1.6465\n",
      "Epoch [13/20], Step [62/398], Loss: 1.6385\n",
      "Epoch [13/20], Step [112/398], Loss: 1.6233\n",
      "Epoch [13/20], Step [162/398], Loss: 1.6139\n",
      "Epoch [13/20], Step [212/398], Loss: 1.6382\n",
      "Epoch [13/20], Step [262/398], Loss: 1.6013\n",
      "Epoch [13/20], Step [312/398], Loss: 1.6315\n",
      "Epoch [13/20], Step [362/398], Loss: 1.5978\n",
      "Epoch [14/20], Step [13/398], Loss: 1.6066\n",
      "Epoch [14/20], Step [63/398], Loss: 1.5972\n",
      "Epoch [14/20], Step [113/398], Loss: 1.5987\n",
      "Epoch [14/20], Step [163/398], Loss: 1.5956\n",
      "Epoch [14/20], Step [213/398], Loss: 1.5759\n",
      "Epoch [14/20], Step [263/398], Loss: 1.5669\n",
      "Epoch [14/20], Step [313/398], Loss: 1.5842\n",
      "Epoch [14/20], Step [363/398], Loss: 1.5532\n",
      "Epoch [15/20], Step [14/398], Loss: 1.5623\n",
      "Epoch [15/20], Step [64/398], Loss: 1.5685\n",
      "Epoch [15/20], Step [114/398], Loss: 1.5478\n",
      "Epoch [15/20], Step [164/398], Loss: 1.5513\n",
      "Epoch [15/20], Step [214/398], Loss: 1.5311\n",
      "Epoch [15/20], Step [264/398], Loss: 1.5543\n",
      "Epoch [15/20], Step [314/398], Loss: 1.5310\n",
      "Epoch [15/20], Step [364/398], Loss: 1.5382\n",
      "Epoch [16/20], Step [15/398], Loss: 1.5259\n",
      "Epoch [16/20], Step [65/398], Loss: 1.5166\n",
      "Epoch [16/20], Step [115/398], Loss: 1.5313\n",
      "Epoch [16/20], Step [165/398], Loss: 1.5343\n",
      "Epoch [16/20], Step [215/398], Loss: 1.5062\n",
      "Epoch [16/20], Step [265/398], Loss: 1.5096\n",
      "Epoch [16/20], Step [315/398], Loss: 1.5039\n",
      "Epoch [16/20], Step [365/398], Loss: 1.4914\n",
      "Epoch [17/20], Step [16/398], Loss: 1.4752\n",
      "Epoch [17/20], Step [66/398], Loss: 1.4991\n",
      "Epoch [17/20], Step [116/398], Loss: 1.4805\n",
      "Epoch [17/20], Step [166/398], Loss: 1.4740\n",
      "Epoch [17/20], Step [216/398], Loss: 1.4892\n",
      "Epoch [17/20], Step [266/398], Loss: 1.4831\n",
      "Epoch [17/20], Step [316/398], Loss: 1.4580\n",
      "Epoch [17/20], Step [366/398], Loss: 1.4673\n",
      "Epoch [18/20], Step [17/398], Loss: 1.4510\n",
      "Epoch [18/20], Step [67/398], Loss: 1.4762\n",
      "Epoch [18/20], Step [117/398], Loss: 1.4542\n",
      "Epoch [18/20], Step [167/398], Loss: 1.4259\n",
      "Epoch [18/20], Step [217/398], Loss: 1.4519\n",
      "Epoch [18/20], Step [267/398], Loss: 1.4555\n",
      "Epoch [18/20], Step [317/398], Loss: 1.4342\n",
      "Epoch [18/20], Step [367/398], Loss: 1.4308\n",
      "Epoch [19/20], Step [18/398], Loss: 1.4064\n",
      "Epoch [19/20], Step [68/398], Loss: 1.4258\n",
      "Epoch [19/20], Step [118/398], Loss: 1.3984\n",
      "Epoch [19/20], Step [168/398], Loss: 1.4335\n",
      "Epoch [19/20], Step [218/398], Loss: 1.4157\n",
      "Epoch [19/20], Step [268/398], Loss: 1.4231\n",
      "Epoch [19/20], Step [318/398], Loss: 1.4132\n",
      "Epoch [19/20], Step [368/398], Loss: 1.4104\n",
      "Epoch [20/20], Step [19/398], Loss: 1.4181\n",
      "Epoch [20/20], Step [69/398], Loss: 1.4096\n",
      "Epoch [20/20], Step [119/398], Loss: 1.3718\n",
      "Epoch [20/20], Step [169/398], Loss: 1.3917\n",
      "Epoch [20/20], Step [219/398], Loss: 1.3731\n",
      "Epoch [20/20], Step [269/398], Loss: 1.3876\n",
      "Epoch [20/20], Step [319/398], Loss: 1.3819\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 726 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 2.6919\n",
      "Epoch [1/20], Step [100/398], Loss: 2.6706\n",
      "Epoch [1/20], Step [150/398], Loss: 2.6547\n",
      "Epoch [1/20], Step [200/398], Loss: 2.6331\n",
      "Epoch [1/20], Step [250/398], Loss: 2.6127\n",
      "Epoch [1/20], Step [300/398], Loss: 2.5941\n",
      "Epoch [1/20], Step [350/398], Loss: 2.5750\n",
      "Epoch [2/20], Step [1/398], Loss: 2.5596\n",
      "Epoch [2/20], Step [51/398], Loss: 2.5422\n",
      "Epoch [2/20], Step [101/398], Loss: 2.5213\n",
      "Epoch [2/20], Step [151/398], Loss: 2.5114\n",
      "Epoch [2/20], Step [201/398], Loss: 2.4920\n",
      "Epoch [2/20], Step [251/398], Loss: 2.4737\n",
      "Epoch [2/20], Step [301/398], Loss: 2.4639\n",
      "Epoch [2/20], Step [351/398], Loss: 2.4384\n",
      "Epoch [3/20], Step [2/398], Loss: 2.4267\n",
      "Epoch [3/20], Step [52/398], Loss: 2.4123\n",
      "Epoch [3/20], Step [102/398], Loss: 2.3930\n",
      "Epoch [3/20], Step [152/398], Loss: 2.3791\n",
      "Epoch [3/20], Step [202/398], Loss: 2.3732\n",
      "Epoch [3/20], Step [252/398], Loss: 2.3599\n",
      "Epoch [3/20], Step [302/398], Loss: 2.3466\n",
      "Epoch [3/20], Step [352/398], Loss: 2.3342\n",
      "Epoch [4/20], Step [3/398], Loss: 2.3171\n",
      "Epoch [4/20], Step [53/398], Loss: 2.2928\n",
      "Epoch [4/20], Step [103/398], Loss: 2.2892\n",
      "Epoch [4/20], Step [153/398], Loss: 2.2744\n",
      "Epoch [4/20], Step [203/398], Loss: 2.2627\n",
      "Epoch [4/20], Step [253/398], Loss: 2.2486\n",
      "Epoch [4/20], Step [303/398], Loss: 2.2479\n",
      "Epoch [4/20], Step [353/398], Loss: 2.2161\n",
      "Epoch [5/20], Step [4/398], Loss: 2.2129\n",
      "Epoch [5/20], Step [54/398], Loss: 2.1925\n",
      "Epoch [5/20], Step [104/398], Loss: 2.1917\n",
      "Epoch [5/20], Step [154/398], Loss: 2.1730\n",
      "Epoch [5/20], Step [204/398], Loss: 2.1649\n",
      "Epoch [5/20], Step [254/398], Loss: 2.1615\n",
      "Epoch [5/20], Step [304/398], Loss: 2.1516\n",
      "Epoch [5/20], Step [354/398], Loss: 2.1373\n",
      "Epoch [6/20], Step [5/398], Loss: 2.1239\n",
      "Epoch [6/20], Step [55/398], Loss: 2.1066\n",
      "Epoch [6/20], Step [105/398], Loss: 2.1105\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 210 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 1.6801\n",
      "Epoch [1/20], Step [100/398], Loss: 1.3891\n",
      "Epoch [1/20], Step [150/398], Loss: 1.2009\n",
      "Epoch [1/20], Step [200/398], Loss: 1.0549\n",
      "Epoch [1/20], Step [250/398], Loss: 1.0046\n",
      "Epoch [1/20], Step [300/398], Loss: 0.9077\n",
      "Epoch [1/20], Step [350/398], Loss: 0.8743\n",
      "Epoch [2/20], Step [1/398], Loss: 0.8220\n",
      "Epoch [2/20], Step [51/398], Loss: 0.7902\n",
      "Epoch [2/20], Step [101/398], Loss: 0.7293\n",
      "Epoch [2/20], Step [151/398], Loss: 0.7156\n",
      "Epoch [2/20], Step [201/398], Loss: 0.7119\n",
      "Epoch [2/20], Step [251/398], Loss: 0.6462\n",
      "Epoch [2/20], Step [301/398], Loss: 0.6482\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [2/20], Step [351/398], Loss: 0.6268\n",
      "Epoch [3/20], Step [2/398], Loss: 0.6061\n",
      "Epoch [3/20], Step [52/398], Loss: 0.6357\n",
      "Epoch [3/20], Step [102/398], Loss: 0.5767\n",
      "Epoch [3/20], Step [152/398], Loss: 0.5872\n",
      "Epoch [3/20], Step [202/398], Loss: 0.5769\n",
      "Epoch [3/20], Step [252/398], Loss: 0.5627\n",
      "Epoch [3/20], Step [302/398], Loss: 0.5750\n",
      "Epoch [3/20], Step [352/398], Loss: 0.5534\n",
      "Epoch [4/20], Step [3/398], Loss: 0.5417\n",
      "Epoch [4/20], Step [53/398], Loss: 0.5320\n",
      "Epoch [4/20], Step [103/398], Loss: 0.5502\n",
      "Epoch [4/20], Step [153/398], Loss: 0.5403\n",
      "Epoch [4/20], Step [203/398], Loss: 0.5182\n",
      "Epoch [4/20], Step [253/398], Loss: 0.5280\n",
      "Epoch [4/20], Step [303/398], Loss: 0.5319\n",
      "Epoch [4/20], Step [353/398], Loss: 0.5183\n",
      "Epoch [5/20], Step [4/398], Loss: 0.4884\n",
      "Epoch [5/20], Step [54/398], Loss: 0.4891\n",
      "Epoch [5/20], Step [104/398], Loss: 0.5065\n",
      "Epoch [5/20], Step [154/398], Loss: 0.4940\n",
      "Epoch [5/20], Step [204/398], Loss: 0.4803\n",
      "Epoch [5/20], Step [254/398], Loss: 0.4850\n",
      "Epoch [5/20], Step [304/398], Loss: 0.4882\n",
      "Epoch [5/20], Step [354/398], Loss: 0.4893\n",
      "Epoch [6/20], Step [5/398], Loss: 0.4679\n",
      "Epoch [6/20], Step [55/398], Loss: 0.4734\n",
      "Epoch [6/20], Step [105/398], Loss: 0.4516\n",
      "Epoch [6/20], Step [155/398], Loss: 0.4735\n",
      "Epoch [6/20], Step [205/398], Loss: 0.4579\n",
      "Epoch [6/20], Step [255/398], Loss: 0.4691\n",
      "Epoch [6/20], Step [305/398], Loss: 0.4900\n",
      "Epoch [6/20], Step [355/398], Loss: 0.4576\n",
      "Epoch [7/20], Step [6/398], Loss: 0.4582\n",
      "Epoch [7/20], Step [56/398], Loss: 0.4507\n",
      "Epoch [7/20], Step [106/398], Loss: 0.4498\n",
      "Epoch [7/20], Step [156/398], Loss: 0.4536\n",
      "Epoch [7/20], Step [206/398], Loss: 0.4355\n",
      "Epoch [7/20], Step [256/398], Loss: 0.4295\n",
      "Epoch [7/20], Step [306/398], Loss: 0.4566\n",
      "Epoch [7/20], Step [356/398], Loss: 0.4460\n",
      "Epoch [8/20], Step [7/398], Loss: 0.4445\n",
      "Epoch [8/20], Step [57/398], Loss: 0.4615\n",
      "Epoch [8/20], Step [107/398], Loss: 0.4247\n",
      "Epoch [8/20], Step [157/398], Loss: 0.4228\n",
      "Epoch [8/20], Step [207/398], Loss: 0.4313\n",
      "Epoch [8/20], Step [257/398], Loss: 0.4528\n",
      "Epoch [8/20], Step [307/398], Loss: 0.4382\n",
      "Epoch [8/20], Step [357/398], Loss: 0.4438\n",
      "Epoch [9/20], Step [8/398], Loss: 0.4438\n",
      "Epoch [9/20], Step [58/398], Loss: 0.4290\n",
      "Epoch [9/20], Step [108/398], Loss: 0.4422\n",
      "Epoch [9/20], Step [158/398], Loss: 0.4185\n",
      "Epoch [9/20], Step [208/398], Loss: 0.4279\n",
      "Epoch [9/20], Step [258/398], Loss: 0.4440\n",
      "Epoch [9/20], Step [308/398], Loss: 0.4201\n",
      "Epoch [9/20], Step [358/398], Loss: 0.4309\n",
      "Epoch [10/20], Step [9/398], Loss: 0.4251\n",
      "Epoch [10/20], Step [59/398], Loss: 0.4173\n",
      "Epoch [10/20], Step [109/398], Loss: 0.4431\n",
      "Epoch [10/20], Step [159/398], Loss: 0.4287\n",
      "Epoch [10/20], Step [209/398], Loss: 0.4151\n",
      "Epoch [10/20], Step [259/398], Loss: 0.4131\n",
      "Epoch [10/20], Step [309/398], Loss: 0.4411\n",
      "Epoch [10/20], Step [359/398], Loss: 0.4238\n",
      "Epoch [11/20], Step [10/398], Loss: 0.4355\n",
      "Epoch [11/20], Step [60/398], Loss: 0.4060\n",
      "Epoch [11/20], Step [110/398], Loss: 0.4309\n",
      "Epoch [11/20], Step [160/398], Loss: 0.4358\n",
      "Epoch [11/20], Step [210/398], Loss: 0.4368\n",
      "Epoch [11/20], Step [260/398], Loss: 0.3922\n",
      "Epoch [11/20], Step [310/398], Loss: 0.4207\n",
      "Epoch [11/20], Step [360/398], Loss: 0.3807\n",
      "Epoch [12/20], Step [11/398], Loss: 0.4242\n",
      "Epoch [12/20], Step [61/398], Loss: 0.4121\n",
      "Epoch [12/20], Step [111/398], Loss: 0.4228\n",
      "Epoch [12/20], Step [161/398], Loss: 0.4016\n",
      "Epoch [12/20], Step [211/398], Loss: 0.4008\n",
      "Epoch [12/20], Step [261/398], Loss: 0.3978\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 439 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 1.7014\n",
      "Epoch [1/20], Step [100/398], Loss: 1.3939\n",
      "Epoch [1/20], Step [150/398], Loss: 1.2028\n",
      "Epoch [1/20], Step [200/398], Loss: 1.0816\n",
      "Epoch [1/20], Step [250/398], Loss: 1.0132\n",
      "Epoch [1/20], Step [300/398], Loss: 0.9451\n",
      "Epoch [1/20], Step [350/398], Loss: 0.8853\n",
      "Epoch [2/20], Step [1/398], Loss: 0.8484\n",
      "Epoch [2/20], Step [51/398], Loss: 0.8267\n",
      "Epoch [2/20], Step [101/398], Loss: 0.7894\n",
      "Epoch [2/20], Step [151/398], Loss: 0.7750\n",
      "Epoch [2/20], Step [201/398], Loss: 0.7737\n",
      "Epoch [2/20], Step [251/398], Loss: 0.7495\n",
      "Epoch [2/20], Step [301/398], Loss: 0.7539\n",
      "Epoch [2/20], Step [351/398], Loss: 0.7425\n",
      "Epoch [3/20], Step [2/398], Loss: 0.7158\n",
      "Epoch [3/20], Step [52/398], Loss: 0.7412\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 101 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 1.8074\n",
      "Epoch [1/20], Step [100/398], Loss: 1.7216\n",
      "Epoch [1/20], Step [150/398], Loss: 1.6712\n",
      "Epoch [1/20], Step [200/398], Loss: 1.6478\n",
      "Epoch [1/20], Step [250/398], Loss: 1.6502\n",
      "Epoch [1/20], Step [300/398], Loss: 1.6549\n",
      "Epoch [1/20], Step [350/398], Loss: 1.6328\n",
      "Epoch [2/20], Step [1/398], Loss: 1.6411\n",
      "Epoch [2/20], Step [51/398], Loss: 1.6430\n",
      "Epoch [2/20], Step [101/398], Loss: 1.6390\n",
      "Epoch [2/20], Step [151/398], Loss: 1.6380\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 74 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 0.4847\n",
      "Epoch [1/20], Step [100/398], Loss: 0.4399\n",
      "Epoch [1/20], Step [150/398], Loss: 0.4327\n",
      "Epoch [1/20], Step [200/398], Loss: 0.4146\n",
      "Epoch [1/20], Step [250/398], Loss: 0.4243\n",
      "Epoch [1/20], Step [300/398], Loss: 0.3945\n",
      "Epoch [1/20], Step [350/398], Loss: 0.3911\n",
      "Epoch [2/20], Step [1/398], Loss: 0.4353\n",
      "Epoch [2/20], Step [51/398], Loss: 0.3741\n",
      "Epoch [2/20], Step [101/398], Loss: 0.3753\n",
      "Epoch [2/20], Step [151/398], Loss: 0.3791\n",
      "Epoch [2/20], Step [201/398], Loss: 0.4087\n",
      "Epoch [2/20], Step [251/398], Loss: 0.3750\n",
      "Epoch [2/20], Step [301/398], Loss: 0.3867\n",
      "Epoch [2/20], Step [351/398], Loss: 0.3921\n",
      "Epoch [3/20], Step [2/398], Loss: 0.4039\n",
      "Epoch [3/20], Step [52/398], Loss: 0.3941\n",
      "Epoch [3/20], Step [102/398], Loss: 0.3766\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 105 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 0.7253\n",
      "Epoch [1/20], Step [100/398], Loss: 0.7143\n",
      "Epoch [1/20], Step [150/398], Loss: 0.7146\n",
      "Epoch [1/20], Step [200/398], Loss: 0.7224\n",
      "Epoch [1/20], Step [250/398], Loss: 0.7490\n",
      "Epoch [1/20], Step [300/398], Loss: 0.7240\n",
      "Epoch [1/20], Step [350/398], Loss: 0.7399\n",
      "Epoch [2/20], Step [1/398], Loss: 0.7668\n",
      "Epoch [2/20], Step [51/398], Loss: 0.7101\n",
      "Epoch [2/20], Step [101/398], Loss: 0.7092\n",
      "Epoch [2/20], Step [151/398], Loss: 0.7081\n",
      "Epoch [2/20], Step [201/398], Loss: 0.7522\n",
      "Epoch [2/20], Step [251/398], Loss: 0.7172\n",
      "Epoch [2/20], Step [301/398], Loss: 0.7321\n",
      "Epoch [2/20], Step [351/398], Loss: 0.7423\n",
      "Epoch [3/20], Step [2/398], Loss: 0.7460\n",
      "Epoch [3/20], Step [52/398], Loss: 0.7352\n",
      "Epoch [3/20], Step [102/398], Loss: 0.7350\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 106 sec \n",
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/20], Step [50/398], Loss: 1.6676\n",
      "Epoch [1/20], Step [100/398], Loss: 1.6810\n",
      "Epoch [1/20], Step [150/398], Loss: 1.6388\n",
      "Epoch [1/20], Step [200/398], Loss: 1.6586\n",
      "Epoch [1/20], Step [250/398], Loss: 1.6638\n",
      "Epoch [1/20], Step [300/398], Loss: 1.6708\n",
      "Epoch [1/20], Step [350/398], Loss: 1.6730\n",
      "Epoch [2/20], Step [1/398], Loss: 1.6881\n",
      "Epoch [2/20], Step [51/398], Loss: 1.6586\n",
      "Epoch [2/20], Step [101/398], Loss: 1.6481\n",
      "Epoch [2/20], Step [151/398], Loss: 1.6749\n",
      "Epoch [2/20], Step [201/398], Loss: 1.6763\n",
      "Epoch [2/20], Step [251/398], Loss: 1.6444\n",
      "Epoch [2/20], Step [301/398], Loss: 1.6734\n",
      "Epoch [2/20], Step [351/398], Loss: 1.6345\n",
      "Epoch [3/20], Step [2/398], Loss: 1.6728\n",
      "Epoch [3/20], Step [52/398], Loss: 1.6601\n",
      "Epoch [3/20], Step [102/398], Loss: 1.6593\n",
      "no improvement in accuracy for 10 iterations\n",
      "Model is trained in 105 sec \n"
     ]
    }
   ],
   "source": [
    "\n",
    "#hyper-params\n",
    "batch_size = 5120\n",
    "optim = 'Adam'\n",
    "\n",
    "input_dim = X_train.shape[1]\n",
    "num_class = len(np.unique(y_train))\n",
    "\n",
    "accuracies = {}\n",
    "learning_rates = [1e-4,1e-2,1e-0]\n",
    "regularizations = [1e-6,1e-4,1e-2]\n",
    "best_model = None\n",
    "best_acc = -1\n",
    "num_layers = 3\n",
    "method = 'softmax'\n",
    "num_epochs = 20\n",
    "for lr in learning_rates:\n",
    "    for reg in regularizations:\n",
    "\n",
    "        classifier_args = (method,optim,lr,reg,batch_size,input_dim,num_class,num_epochs)\n",
    "        config =  '{}/train/optim_{}_lr_{}_reg_{}_bs_{}'.format(method,num_layers,optim,lr,reg,batch_size)\n",
    "        runs_dir = join(dataroot,'runs',config)\n",
    "        \n",
    "        X_train = X_train.astype(float)\n",
    "        y_train = y_train.astype(int)\n",
    "        p = np.random.permutation(len(y_train))\n",
    "        X_train = X_train[p]\n",
    "        y_train = y_train[p]\n",
    "        X_train,y_train = balance_data(X_train,y_train,seed=SEED)\n",
    "\n",
    "        tick = time.time()\n",
    "        clf = getClassifier(classifier_args,runs_dir)\n",
    "        \n",
    "        clf.fit(X_train,y_train)\n",
    "        pred = clf.predict(X_test)\n",
    "        \n",
    "        acc = metrics.balanced_accuracy_score(y_test,pred)\n",
    "        if acc >best_acc:\n",
    "            best_model = clf\n",
    "            best_acc = acc\n",
    "        accuracies[(lr,reg)]=acc\n",
    "        tock = time.time()\n",
    "        print(\"Model is trained in {0:.0f} sec \".format(tock-tick))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(0.0001, 1e-06) -> 60.83\n",
      "\n",
      "(0.0001, 0.0001) -> 60.38\n",
      "\n",
      "(0.0001, 0.01) -> 47.48\n",
      "\n",
      "(0.01, 1e-06) -> 80.50\n",
      "\n",
      "(0.01, 0.0001) -> 66.94\n",
      "\n",
      "(0.01, 0.01) -> 54.83\n",
      "\n",
      "(1.0, 1e-06) -> 78.96\n",
      "\n",
      "(1.0, 0.0001) -> 67.60\n",
      "\n",
      "(1.0, 0.01) -> 50.63\n"
     ]
    }
   ],
   "source": [
    "# accuracies for CV\n",
    "for x in accuracies:\n",
    "    print()\n",
    "    print('{0:} -> {1:.2f}'.format(x,accuracies[x]*100))\n",
    "results = accuracies  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEWCAYAAABv+EDhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZxcVZ338c+3qruz7wlbNgIEZI/QgoAwoBiWR8kou4owDuIWnEcdEGQeQFCHEUecGXHGiBEXNjcgIkMIIqgokATZwhoCIYkJZIMkJOl0d/2eP84puCmqum93V3V3Vf3er9d9dd31/Kqr6p57zzn3HJkZzjnn6k+mrwNwzjnXNzwDcM65OuUZgHPO1SnPAJxzrk55BuCcc3XKMwDnnKtTngEUkLRJ0m79II4jJT1bxuPdJOnvy3W8eibpV5JOqNCxr5f0tfi6w+9ActtuptUvvuuu71RVBiDpJUmvShqSWHaupPtS7t/pD8bMhprZkpTHM0l7pNm2q8zsj2a2VzmOJekA4EDg9sSynSX9UNJKSRslPSPpq/n/beF7k7SnpF9IWiPpdUmPS/qipGyJNO+TdG58fbSkXDzhbJK0XNLPJb2rYJ8Zkh6VtCGmc6+kKSWOv6ek2yWtlrRO0lxJexVs8wVJq+LxZksakFh3paQnJLVJurxgv68kYt0kaUuMf2zc5N+Abp940yrzd+DNzyNx/NTfdVebqioDiLLAP/V1EGlIaujrGKJPATdYfOpP0mjgL8Ag4DAzGwa8HxgJ7F64s6TdgYeAZcD+ZjYCOBVoBoaljOFvZjY0bv9u4Bngj5LeF9PYA/gJ8CVgBDAFuBZoL3G8kcAcYC9gR+Bhts/gjgMuAt4HTAZ2A76a2H8xcCHw28IDm9k34slxaIz534D7zGxNXP8wMFxSc8r37nqJgmo8r/UNM6uaCXiJ8KNeB4yMy84l/Djz27wDmBe3eRY4LS4/D2gFtgGbgN+USMOAPeLr6wknod8CGwknwd3juj/Ebd+IxzsdOBpYDnwZWAX8FDgH+FMHaZwIPBWPvwL457j8aGB5Yp+9gfuA14BFwEmJdSXjjOuXAO9JzH8NeALIdPC/Tsb4M+C3Xfys7gPOLfZeEtt8F1gQX58CPNqD78boGPOYOH8j8I3E+vcBq4rs9zPg8g6Oq/j/O7tg+Q+Ay0rs8zTwgcR8A7AaOCjO/yJ+P16P36N9Cz7Lr5X4DrwTeCR+xrcANye2HQXcEdNZH19PiOu+TshIt8bv6neLfMYjCBnwamAp8C/57wfxOwx8Kx77ReCEDv5nFwEvxDifAj5UsP6T8X+UX5//v0wEfh1jWJuI83LgZ4n9d42xNyS+a18HHgC2AHsA/5BIYwnwqYIYZgCPAhtirMcTLmoWFmz3ReD27n4v+/tUjTnlAsIH/s+FK2LxxTzCj38H4Azge5L2MbNZwA3ANy1c2X0wZXpnEK4cRxGuGr8OYGZHxfUHxuPdEud3IpyMJhMync78kPDlHAbsB9xb5H01Ar8B7o7v63zghoIij6Jxxv/JFEJmmHcs8Gszy6WIL7/9L1Nu2xW/Bg6KMT4CvEPSNZKOkTS0i8c6inCCXxvn9wUeS6x/DNhR0pguHvdIwv/8VwXLnyYUqxVzE3BmYv44YI2ZPRLn/xeYGo/7COF72SFJTcBthIuK0YRM5OTEJhngR4Tv3STCifC7AGZ2CfBHYGb8rs4sksR/ETKB3YC/Az5OOInmHUr4Do0Fvgn8UJJKhPsC4f82gvCd/JmkneP7OJVwQv84MBw4CVgbixLvIGQ+uwLjCRlcWmcRfm/D4jFeBT4Q0/gH4BpJB8UYDiFkdhcQ7iSPIlxczgGmSNq74Lg/6UIcVaUaMwCAS4HzJY0rWP4B4CUz+5GZtZnZXwk/3FN7kNatZvawmbURfqjTOtk+R7gybDGzLSmO3wrsI2m4ma1PnCSS3g0MBa4ys21mdi/hx5I8yZSKc2T8uzGx7RhgZYrYurt9Wn8jXGGPtFAWfTThh/9zYE2ss+k0I5A0gXAH9MXE4qGEK+y8/Ou0RVZ5ZwO/NLNNBcs38tb/ttCNwEmSBsf5jxAyBQDMbLaZbTSzFsLJ8EBJIzqJ491AI/AdM2s1s18C8xPHXGtmvzKzzWa2kXAB8Hdp3mA8+Z4BXBzjegn4d8LJL2+pmf3AzNqBHwM7E4re3sbMfmFmfzOzXLwweh44JK4+l3ARNt+CxWa2NK7fBbjAzN4ws61m9qc08UfXm9mi+LtvNbPfmtkLMY37CRdPR8Zt/xGYbWbzYowrzOyZ+HncAnws/l/2JWRGd3QhjqpSlRmAmT1J+FAuKlg1GThU0mv5Cfgo4aq8u1YlXm8mnFg6strMtnbh+CcTioGWSrpf0mFFttkFWFZwxb6UcLLsLM7X4t/kiW8t4QecVofbS/qfRIXpV7pw3PGEW/nXAMzsQTM7zczGEX6sRwGXxDSSlbKTEmmPI/y4v2dmNyWOvYlw9ZeXf53MCDsUT+CnEk54hYbx1v92O2a2mHCH8MF4jJMImQKSspKukvSCpA2EK08IV9Yd2QVYYWbJ3huXJmOV9H1JS+Nx/wCMLFVJX2AsIXNZmlhW8vtlZpvjy6K/BUkfj5X5+d/gfrz1/iYS7hAKTSRkMm0p4i1mWUEMJ0h6MDYQeI3wG+ssBgif9Ufi3c1ZwM9jxlCTqjIDiC4jlCUmv6TLgPvNbGRiGmpmn4nre6Pr08I03gDyV4JI2i4zildCMwjFAbcRrn4L/Q2YWFC5NYlQZ9BxMGZvEL7seyYW3wN8qAuVZfewfXFDYRqftrcqTb+R8pgAHwIeiTEWHnM+oYhovzg/NDG9DCBpFOHkP8fMvl5wiEVsX0RzIPBKoogobXzrCEWOhfZm+yKmQvlioBnAUzFTgHA3MINQrDaCcIUJ4U6oIyuB8QXFLpMSr79EqBA/1MyGEzLP5HE7+u6vIdyJTi44dqffr0KSJhPqR2YS6mNGAk8m4lhGkYYGcfmkEg0ntvsNUfyC7s33F1t7/YpQZ7FjjOHOFDFgZg8S6gmPJHxWPy22Xa2o2gwg/qBuAT6fWHwHsKeksyQ1xuldiTK9VwhlnOWS5niPAftKmiZpIOGWHwjlupI+KmmEmbUSKqSKlcs/RLiqvzC+p6OBD5K+jPROti8O+DbhivjH8QeLpPGSvq3QZLTQZcDhkq7OZ2CS9pD0M0mlikGKiq00xku6jFAc8JW4/D2SPilphzj/DsKV84MljjMcmAs8YGaFd4IQym3/UdI+McZ/IVSw5vdvjJ9HBmiQNLDI1fLZwE8Krrrz/o5Qll/KzcB04DPEq/9oGNBCuKsaDKTNMP8CtAGfj7F/mLeKVfLH3QK8ptDK67KC/Ut+V2Oxzs+Br0saFr8TXyRUkHfVEMLJeDWApH8gZuLRdcA/Szo4fhf2iOk9TMjkrpI0JH4eR8R9HgWOkjQpFpVd3EkMTcCAGEObwjMb0xPrfwj8g6T3ScrE7+M7Eut/Qqg/ae1iMVTVqdoMILqC8IUDIJZ9TieUZ/6NcNv6b4QvA4QPfp94a3pbGdK/nHASfU3SacU2MLPnYpz3EMpCC79QZwEvxdv2TxOKrAqPsY1wwj+BcLX2PeDjZvZMyjhnAR/NXz2a2TrgcMJV30OSNgK/I5STLy7c2cxeAA4jXK0ukvQ64QprAemLVHaRtIlQNDMf2B842szujutfI5zwn4jb3QXcSqhwLOZDwLsIP+S3FQ+Z2V1x398DLxOKNJInxR8QTphnEoqZtpAo85Y0HngvRSoAFZ5f2GShOWhRZraScNI+nHChkveTGMsKQguYohlckeNtAz5MaJGzjtDq7NeJTb5DaNa7Jh7zroJD/AdwiqT1kv6zSBLnE660lxC+ozcCs9PEVhDnU4T6g78QMp39Ca1z8ut/QaifuJHw3bkNGB0zoQ8SWvC8TGhNd3rcZx7hf/g4sJBOyuTjeeDzhExtPeFKfk5i/cPEimHCd/5+tr/7+Skh0+pOBlhVVPzixtUaSTcSyjPLkfHVNUm/An5oZnf2dSyu/CQNIrQiOsjMnu/reCrJMwDnnEuQ9EXCcxzv7etYKq2/PKnqnHN9TtJLhMriuug3y+8AnHOuTlV7JbBzzrluqskioLFjx9quu+7a12E45/q5hQsXrokPHnbbwZkhtsFK9Vm4vcW0zDWz4zvaRtLxhFZbWeA6M7uqYP0kwgNrI+M2F+UbJEi6mPCkczvweTOb21FaNZkB7LrrrixYsKCvw3DO9XOSlna+Vcc2WDvfaZjc+YbAB9qe6/CJ7/gsyrWE3nmXA/MlzYnNa/P+hdCi778l7UN4zmfX+PoMQj9YuwD3SNozNrEtyouAnHOuJwRqVKophUOAxWa2JD77cTPhyfEk462uTUYQnnkibnezhX7IXiQ803MIHajJOwDnnOstyojsoDRdLhEeN+zYeLbv12g5oSfWpMuBuyWdT3gQ9tjEvskHC5ezfVc5b+N3AM451xOCTINSTcBYSQsSU5ou4wudSej9dAKhk7ufdqFfr+3U/R3A1q3tvLahlcGDsgwf1tjX4bgKaGk1tmyDQU0wIN1tuKsymza3s6XFGD4kw4CmXr6ujUVAKa0xs45GkltB6K00bwJv75TvHwkD2GBmf4l9Wo1Nue926jYDWLL0DX5000v86eG1ZDOird3YbfIQzj59Mn93WGc987pqsPRV4w9PtvPyasgK2g0mj4Oj9ssyaQfPCKqdmfHwk1u49Xevs+KVVrIZkTPj4H0G8+FjhzNxp6ZeiUN68+q+HOYDUxXGwl5BqNT9SME2LxNGuLs+dnQ5kNDx3RzgRknfJlQCTyV0sldSXWYA8/+6jou/voiWbTnMoDX2JPvcC5u48t+f5rHjdubzn6zIWO+ulyx4vp15fzXaYvuHfBerL74Cy9e0M/0gcdAeKcttXb9jZvz49vXct+ANWraF329be/j70BObeeTpLVxwzjj2mzqw8sF07Q6gQ2bWJmkmoafbLGHgmkWSriAMnzqH0PX3DyR9gVAhfE7ssXaRpJ8TOhlsAz7XUQsg6KM6gNit8DOSHpd0a6kuhSUdL+lZSYslFevyt8tee72Vr3xjEVtbwsm/0NaWHHPmruT3D6wuR3KuD6xcZ9ud/Au1tsPcR4xV6/0p+Gr14GObtzv5J5nBtlbjWz9ezcY30rXP75Gu1QF0yszuNLM9zWz3/DgXZnZpPPljZk+Z2RFmdqCZTUv0qIuZfT3ut5eZddRdOdB3lcDzgP3M7ADgOYr0751oD3sCsA9wZmzn2iNz7l5JrpORcLe25PjxLT1uHuz6yJ+fztHeyWfcnoO/PJ12SGTX3/z6dxuKnvyTzOC++YUjeZafBNmmTKqpv+mTiMzs7sTQbw8SKisKpWkP22Vz732Flm2d//CXLt/M+te29TQ51weeW2FF7+6SzODZ5X4HUI1e29jOqrWtnW63rdX4w8LNnW7Xc0KZdFN/0x+ypE9QfGSlYu1hS7ZplXRevmnV6tWli282b0k35GhDNsPmLb1w++jKyqx00U+hNr8BqEpbtuZoSHky3drSCx+yQNlMqqm/qVhEku6R9GSRaUZim0sIlRU39DQ9M5tlZs1m1jxuXOmuPcaNGVByXVJbe46RI7xZaLWRxKCUjT/Sbuf6lxFDs29W+HZm9IjKV/QLyGSVaupvKpYBmNmxZrZfkel2AEnnAB8APlpizNUut2lN45QPjmfQwI6/FBIcetBohgyuy0ZSVe+du4vOLrayGTh4j/73g3SdGzwok6p1z8Amcfx7hlU+IOFFQF0Re7u7EDjJzEoV0r3ZHlZSE6E97JwS26Z29BHjGD6sgUwH77ypKcMnzkzXuZPrfw7ZM0NDJxd+DVlontr/bsldOqe8fwRNHTS9zAiGDcnwrn0HVzwWSV4J3EXfBYYB8yQ9Kul/ACTtIulOCO1hgXx72KcJvd8t6mnCTY0Zrr1qGuPGDGDQwO3fflNThoEDMlxx4T7suXsvXDm4ihg2WJz13iwDG6GxICNozMLARjjrvVmGDup/V2Qund0nDuD8j4yhqVE0FtyoD2wSo0dmuewzO9JQvge0OqRMJtXU39TkiGDNzc3WWXfQ21pz3PfAan5++3JeXbuNQQMzHHf0jsw4YRfGjPLC4VqwdZvx+Is5Fi4OXUEMboKDp4oDpmS8S4gasX5DO/c8uJEH/rqZrdtyjBnRwIlHDuOQ/QfTmOLkL2lhJ10zdGqf4UPtZ++almrbg+99oMfplVPdFnI3NWaYfvSOTD96x74OxVXIwCZxyF5ZDtmrryNxlTJqeJZTp4/k1OlFnyXtJf2zgjeNus0AnHOuHBQrgauRZwDOOddD/bF8Pw3PAJxzrickso2eATjnXN3xIiDnnKtjXgTknHP1yO8AnHOuXvXPbh7S8AzAOed6QIJMZ32P9FOeATjnXA/5g2DOOVeP5EVAzjlXt7wVkHPO1SF/DsA55+qYZwDOOVePJG8F5Jxz9UlVWwfQV0NCXi3pGUmPS7pVUtHOvCW9JOmJOGpYxyO8OOdcX5HSTakOpeMlPStpsaSLiqy/Jp4TH5X0nKTXEuvaE+s6HUK3r+4A5gEXm1mbpH8DLga+XGLbY8xsTe+F5pxz6ZWzElhSFrgWeD+wHJgvaY6ZPZXfxsy+kNj+fOCdiUNsMbN0w5PRR3cAZnZ3HPMX4EFgQl/E4Zxz5VDGMYEPARab2RIz2wbcDMzoYPszgZu6G3d/KLj6BPC/JdYZcLekhZLO6+ggks6TtEDSgtWrV5c9SOecKyo+CJZmSmE8sCwxvzwuK5KsJgNTgHsTiwfG8+CDkv6+s8QqVgQk6R5gpyKrLjGz2+M2lwBtwA0lDvMeM1shaQdgnqRnzOwPxTY0s1nALAiDwvf4DTjnXEpdaAU0tqA+c1Y8d3XHGcAvzaw9sWxyPGfuBtwr6Qkze6HUASqWAZjZsR2tl3QO8AHgfWZW9IRtZivi31cl3Uq4PSqaATjnXF/oYh3AGjNr7mD9CmBiYn5CXFbMGcDnkgsS58wlku4j1A+UzAD6qhXQ8cCFwElmtrnENkMkDcu/BqYDT/ZelM45l4Ygk0k3dW4+MFXSFElNhJP821rzSHoHMAr4S2LZKEkD4uuxwBHAU4X7JvVVK6DvAgMIxToAD5rZpyXtAlxnZicCOwK3xvUNwI1mdlcfxeuccyUpZRPPzsSWkTOBuUAWmG1miyRdASwws3xmcAZwc0Hpyd7A9yXlCBf3VyVbDxXTJxmAme1RYvnfgBPj6yXAgb0Zl3POdZnK2xmcmd0J3Fmw7NKC+cuL7PdnYP+upOVPAjvnXE9IyLuCcM65+uSdwTnnXB0SQuoPj1R1XaoMQNLhwK7J7c3sJxWKyTnnqoeAWr0DkPRTYHfgUSD/wIEBngE45xy1PSJYM7BPqYe1nHOu3tVyHcCThC4dVlY4Fuecqz4SytZYKyBJvyEU9QwDnpL0MNCSX29mJ1U+POecqwI1WAT0rV6LwjnnqpSksj0J3NtKZgBmdj+ApH8zs+0Ga4mDuNxf4dicc646VOkdQJqo319k2QnlDsQ556pVGccD6FUd1QF8BvgssJukxxOrhgEPVDow55yrCqE/6L6Ools6qgO4kTBS178CyYGJN5rZuopG5ZxzVaTmWgGZ2evA65I+V7hOUqOZtVY0Muecqwa1/CQw8AhhhJr1hLc6Elgl6RXgk2a2sILxOedcP6eqfRI4TdTzgBPNbKyZjSFUAN9BqB/4XncTlnSlpMclPSrp7jgYTLHtzpb0fJzO7m56zjlXMVK6qZ9JkwG828zm5mfM7G7gMDN7kDCqV3ddbWYHmNk0QoZyaeEGkkYDlwGHEsYDvkzSqB6k6Zxz5SXKOSRkr0pTBLRS0peBm+P86cArkrJArrsJm9mGxOwQwlPHhY4D5uUrnSXNA44Hbupuus45V1412BVEwkcIV+G3xfkH4rIscFpPEpf0deDjwOvAMUU2GQ8sS8wvj8uKHes84DyASZMm9SQs55xLT9RkM1AAzGwNcH6J1Ys72lfSPYSO5ApdYma3m9klwCWSLgZmEjKabjGzWcAsgObmZu+51DnXS1S7rYAk7Qn8M28fEOa9ne1rZsemjOMGwiDIhRnACuDoxPwE4L6Ux3TOuYoT1PSIYL8A/ge4jrcGhOkxSVPN7Pk4OwN4pshmc4FvJCp+pwMXlysG55zrsRp/DqDNzP67AmlfJWkvQkXyUuDTAJKagU+b2blmtk7SlcD8uM8V/hSyc65/qc2uIPJ+I+mzwK1sPx5Aj07EZnZyieULgHMT87OB2T1JyznnKqpKWwGlybbOBi4A/gwsjNOCSgblnHNVI98ZXJop1eF0vKRnJS2WdFGR9dfEB2gflfScpNcS67r04GyaVkBTUkXtnHP1qkx1APH5qmsJ3fAvB+ZLmmNmT+W3MbMvJLY/H3hnfJ1/cLaZ8FzVwrjv+pJhpwhosKR/kTQrzk+V9IFuvTvnnKtF5bsDOARYbGZLzGwb4QHcGR1sfyZvPRj75oOz8aSff3C2pDQR/QjYBhwe51cAX0uxn3PO1Yf0fQGNlbQgMZ1XcKSuPPw6GZgC3NvVffPSVALvbmanSzoTwMw2q1oHwHTOuXKTutLPzxozay5TymcAvzSzbjfPTxP1NkmDiH31SNqdRGsg55yre5lsuqlzKwjd7+dNiMuKOYPt+0Xryr4h7BQBXQbcBUyUdAPwO+DCFPs551zty98BlKc30PnAVElTJDURTvJz3p6k3gGMAv6SWDwXmC5pVHx4dnpcVlKHRUCxqOcZ4MPAuwnPvP1T7B/IOecclK2vfzNrkzSTcOLOArPNbJGkK4AFZpbPDM4AbjYzS+zb5QdnO8wAzMwk3Wlm+wO/7eZ7cs652lbGJ4HN7E5C32jJZZcWzF9eYt8uPTibJupHJL0r7QGdc66+pGwB1A/bzqRpBXQo8FFJS4E3CMVAZmYHVDQy55yrAiawKu0KIk0GcFzFo3DOuapV253Bfc3MzkoukPRT4KwS2zvnXH2p4Qxg3+RM7Kvi4MqE45xz1cf6Yfl+GiWzLUkXS9oIHCBpQ5w2Aq8Ct/dahM4515+VuTfQ3lQyIjP7VzMbBlxtZsPjNMzMxphZj0blknSlpMdjd6Z3S9qlxHbtiW5P3/YwhHPO9QtV2gooTZZ0h6QhAJI+JunbsROinrjazA4ws2nAHcClJbbbYmbT4nRSD9N0zrkKEJbNppr6mzQZwH8DmyUdCHwJeAH4SU8SNbMNidkhxH6GnHOu6ojaKwJKaIuPG88Avmtm1wLDepqwpK9LWgZ8lNJ3AANjl6kPSvr7nqbpnHOVYMqkmvqbNBFtlHQx8DHgt5IyQGNnO0m6R9KTRaYZAGZ2iZlNBG4AZpY4zOTYdepHgO/EnkhLpXdevo/t1atXp3hbzjlXDrX9JPDphBPwP5rZKkmTgKs728nMjk0Zww2Efi8uK3KMFfHvEkn3EYY+e6FEerOAWQDNzc1epOSc6zX98eo+jU6jNrNVZvZtM/tjnH/ZzHpUByBpamJ2BqHH0cJtRkkaEF+PBY4Anirczjnn+pRUzvEAelWaO4BKuErSXkAOWAp8GkBSM/BpMzsX2Bv4vqQcIaO6KjkwsnPO9QdG9T4I1icZgJmdXGL5AuDc+PrPwP69GZdzznVLlRYB9dUdgHPO1QyjRu8AJB0BXA5Mjtvnu4PerbKhOedcNVDVVgKnuQP4IfAFYCHQ7dHnnXOuZtVwBvC6mf1vxSNxzrkqZBK5ftjCJ400GcDvJV0N/BpoyS80s0cqFpVzzlWTGm4FdGj825xYZsB7yx+Oc85Vn5qtAzCzY3ojEOecq06q6VZAIwjdNBwVF90PXGFmr1cyMOecqxbVegeQJurZwEbgtDhtAH5UyaCcc65qiKrtDC5NBrC7mV1mZkvi9FXAnwFwzjnCQ2A5ZVNNaUg6XtKzkhZLuqjENqdJekrSIkk3JpZ3aRTFNJXAWyS9x8z+FBM4AtiS6p0451wdKFcRkKQscC3wfmA5MF/SnGQ/aLEzzYuBI8xsvaQdEofYEkdaTCVNBvAZ4MexLkDAOuCctAk451ytK2Ml8CHAYjNbAiDpZkKPycmOMD8JXGtm6wHM7NXuJpamFdCjwIGShsf5DZ3s4pxzdaRLXUGMlbQgMT8rjmWSNx5YlphfzltN8fP2BJD0AJAFLjezu+K6gfH4bYQelG/rKJiSGYCkj5nZzyR9sWA5AGb27Y4OXA1yZqzbmKOl1chmYMywLI0N/a+ixnWftbfTtmENubZWMg2NNAwfi/rh4Nyu+6y1hbbXV0N7GxowiOzwcSjTu61yutAd9Jo4ymFPNABTgaOBCcAfJO1vZq8RRlFcIWk34F5JT5hZ0UG08gcqZUj8W2z836oeccvMWL6mjZdebQt9eVuooM9ZK+OGZ9lrfCMNWc8IqplZjq0rFrNtzXJAb33IGE1jJzBw/B6oSpvuucDa29i65DHa168Kn61ZHHxdNE3Yi8Ydd33zgrWicUipK3hTWAFMTMxPiMuSlgMPmVkr8KKk5wgZwvyujKIIHWQAZvb9+PIeM3sguS5WBFetJa+0snxNO7lkNhZfr97QzhstOQ7efQDZjGcC1cjM2PzC47RtXAeWS6wIf7atXk6uZQuDdzugV04QrvysvY3Ni/6Ebd0UTvz533L8vLctexpr3cqAiXv3TjzlqwOYD0yVNIVw4j+DMCRv0m3AmcCP4miJewJLJI0CNptZS2IUxW92lFiaS6D/SrmsWyR9SZLFgIutP1vS83E6u6fpvbE19/aTf4IZbGkxlq1p62lSro+0vfYqbZvWb3/yT7IcbRvXhmIDV5W2rXwB2/pG+MEWk2undeUScls29ko8pkyqqdPjmLUBM4G5wNPAz81skaQrJJ0UN5sLrJX0FPB74AIzW0sYRXGBpMfi8k5HUeyoDuAw4HBgXEE9wHBCxUOPSZoITAdeLrF+NOEp5GZCHr8wNola3900l61pK3nyz8sZLF/TxuRxDX6FWIVaXlkKuU56Ls/laHllKY0jd+h4O9fvmBmtr7xYOoN/c8Mc21a9yMApB1Q+pjJ2BWFmdwJ3Fiy7NPHagC/GKblNl0dR7CEf5wsAABoYSURBVChLagKGEjKJYYlpA3BKVxLpwDXAhZSuUzgOmGdm6+JJfx5wfE8SXL+pky9N1J6DltaqruqoS2ZG++Z0V33tm71BWzWybVsgl/J33At3eRZbAZXjDqC3dVQHcD9wv6TrzWxpuROWNANYYWaPdXCVXaxJ1PgSxzsPOA9g0qRJJdNNfUpXldd017WUn5x/wNWpVLFPH6rZzuCAzXE8gH2BgfmFZtZpd9CS7gF2KrLqEuArhOKfsohtaWcBNDc3l/yGDBuoVFf2AgZ4k9CqI4nMgMHkWjZ3um1mwKBeiMiVm5oGdr5RlBkyooKRvCWXqjq1/0kT9Q3AM8AU4KvAS4Sa6k6Z2bFmtl/hBCyJx3tM0kuEpk6PSCrMLNI0ieqSieMa6KyJsICdR2XJeCugqjRgx8l0+iFnMgzYaddeiceVlzJZGsZNgs6uujNZmnbavTciwsikmvqbNBGNMbMfAq1mdr+ZfYIeDgZjZk+Y2Q5mtquZ7Uoo2jnIzFYVbDoXmC5pVGziND0u67YRgzOMHpqh1LldQGMDTN6hsSfJuD7UOHonMgMGl+59URkyAwbTOKrYzamrBk3jp6KGRkpmAsqSHbEDmaEjKx6LEesBUkz9TZoMoDX+XSnp/0h6JzC6UgFJapZ0HYCZrQOuJNxxzCeMQ7Cuh8dn30lN7DAy+7YeWjOCIQPFwbsPoMmLf6qWMlmG7tlMduioOFh3/rMUKEPD0JEM3bO5158WdeWTaRzAoP2OJDNoKCTH41UmfMZjxzNwj4N6rRVftWYAaeoAvhY7gvsSof3/cOAL5Qwi3gXkXy8Azk3MzyaMSVA2GYm9JzSx247Gqtfa2LrNaMjCDiMaGDbITwq1QNkGhk49iPatm2ldvwprbUGNA2gctRPZgYP7OjxXBpkBgxl8wNG0b3qNtnUrsfZWMgMG0zB2Apku1BOUQ388uaeRJgN4LI7+9TpwDECRsvqqNKBRTB7nRT21LDtwMNmdffiKWpYdOpJsLxT1lNY/r+7TSHO5+6KkmyQlL5vuLLm1c87VEQNylkk19TdpInoC+CPwJ0n5KvXqzO6cc64CarkOwMzse7F/id9I+jL+CI1zzr2pP57c00iTAQjAzB6Q9D7g58A7KhqVc85VDWFWuxnAifkXZrZS0jGETuKcc67uGZCrtTuA/IhgwJkl2tL+oWJROedctTD6ZQVvGt0dEcw551xUc3UAZvZ9SVlgg5ld04sxOedcFaneOoAO71vMrJ0w9JhzzrkiqrkvoDSVwA9I+i5wC/BGfqGZPVKxqJxzropU6x1AmgxgWvx7RWKZ0cMeQZ1zrlakG5+s/+k0AzCzY3ojEOecq0aGarIV0Jsk/R/ePiLYFaX3cM65+lGzRUCS/gcYTOgJ9DrCgPAPVzgu55yrGv2xgjeNNPcth5vZx4H1ZvZV4DBgz3IkLulLkkzS2BLr2yU9Gqc55UjTOefKyiCXcupv0hQBbYl/N0vaBVgL7NzThCVNJAzx+HJHaZvZtA7WO+dcn8o3A61Gae4A7pA0ErgaeIQwKPxNZUj7GuBCvGdR51yVM1OqKQ1Jx0t6VtJiSReV2OY0SU9JWiTpxsTysyU9H6ezO0srTSugK+PLX0m6AxgYRwjrNkkzgBVm9lgnY3YOlLQAaAOuMrPbepKuc86Vn2gvUyVw7H3hWuD9wHJgvqQ5ZvZUYpupwMXAEWa2XtIOcflo4DKgmXBhvTDuu75Ueh11BvfhDtZhZr/u5I3cAxQbOvIS4CuE4p/OTDazFZJ2A+6V9ISZvVAivfOA8wAmTZqU4tDOOddzRllbAR0CLDazJQCSbgZmAE8ltvkkcG3+xG5mr8blxwHzzGxd3HcecDwdlNh0dAfwwQ7WGdBhBmBmxxZbLml/YAqQv/qfADwi6RAzW1VwjBXx7xJJ9wHvBIpmAGY2C5gF0Nzc7MVKzrleY+nPOGNjqUberHjuyhsPLEvMLwcOLTjGngCSHgCywOVmdleJfcd3FExHncH9Q0c7dpeZPQHskJ+X9BLQbGZrkttJGgVsNrOW2EroCOCblYjJOed6oguVwGvMrLmHyTUAU4GjCRfQf4gX1t06UIckXVpseSUeBJPUDHzazM4F9ga+LylHqKy+KlkO5pxz/UJ5m3iuACYm5ifEZUnLgYfMrBV4UdJzhAxhBSFTSO57X0eJpWkG+kbi9UDgA8DTKfZLxcx2TbxeAJwbX/8Z6Fau5pxzvcWAXK5sdQDzgamSphBO6GcAHynY5jZCL80/iqUjewJLCMXj34ilJxDqWS/uKLE0rYD+PTkv6VvA3M7fh3PO1YdyDQlpZm2SZhLOsVlgtpktknQFsMDM5sR10yU9BbQDF5jZWgBJVxIyEYAr8hXCpaTqC6jAYMKthXPOObpUCZziWHYncGfBsksTrw34YpwK950NzE6bVpo6gCd462GtLDCO7buGds65umVVPCJYmjuADyRetwGvmFlbheJxzrnq0k/7+UkjTQawsWB+uKSNsQbaOefqXjmLgHpTmgzgEUKzpPWAgJHAKkmvAJ80s4UVjM855/o1g7J1BdHb0nQGNw840czGmtkY4ATgDuCzwPcqGZxzzlUDs3RTf5MmA3i3mb3Z7NPM7gYOM7MHgQEVi8w556pEtWYAaYqAVkr6MnBznD8deCX2WletYyE751xZmEGuhouAPkJo938bcCuhPuAjhCahp1UuNOecqw41ewcQO2k7X9IQM3ujYPXiyoTlnHPVo71Ky0I6vQOQdHh85PjpOH+gJK/8dc453hoPoFwjgvWmNEVA1xAGGlgLYGaPAUdVMijnnKsaKYt/qrIICMDMlhUM3dhemXCcc6761PKTwMskHQ6YpEbgnyhjd9DOOVfNQhFQX0fRPWmKgD4NfI4wtNgKYFqcd845R/UWAXWYAcS2/meZ2UfNbEcz28HMPpbve7q7JF0uaYWkR+N0Yontjpf0rKTFki7qSZrOOVcRFloBpZn6mw4zADNr5+2j0ZTLNWY2LU53Fq6Mmc+1hK4n9gHOlLRPhWJxzrluCSOCpZv6mzR1AH+S9F3gFhLDQ5rZIxWLKjgEWGxmSwAk3QzMAHxcYOdcv9Ifi3fSSJMBTIt/k4PAGPDeHqY9U9LHgQXAl8xsfcH68cCyxPxy4NBSB5N0HnAewKRJk3oYmnPOpVezGYCZHdOdA0u6B9ipyKpLgP8GriRkJFcC/w58ojvp5JnZLGAWQHNzc5V+HM65amM1PiBMt5jZsWm2k/QDQvfShVYQ+h3KmxCXOedcv2JVeguQphlo2UnaOTH7IeDJIpvNB6ZKmiKpCTgDmNMb8TnnXFe0t6eb+puK3QF04puSphGKgF4CPgUgaRfgOjM70czaJM0E5hJ6Hp1tZov6KF7nnCuqv7bxT6PTDEDSh4ssfh14wsxe7U6iZnZWieV/A05MzN8JvK2JqHPO9Se1XAfwj8BhwO/j/NHAQmCKpCvM7KcVis0556pCtd4BpKkDaAD2NrOTzexkwkNZRmiS+eVKBuecc9XAcpZqSqOzHhAknSNpdaInhXMT69oTyzutM01zBzDRzF5JzL8al62T1JrqHTnnXI0yK183D4keEN5PePZpvqQ5Zlb4AOwtZjazyCG2mNm0IsuLSpMB3CfpDuAXcf6UuGwI8FrahJxzrlblylcJ0Ks9IKQpAvoc8CPCE8HTgB8DnzOzN7r7kJhzztWKfHfQKXsDHStpQWI6r+BwxXpAGF8k2ZMlPS7pl5KSz0sNjMd9UNLfdxZ7mieBTdKfgG3xvT5s1frUg3POlVvXmoGuMbPmHqb4G+AmM2uR9CnCRXm+a57JZrZC0m7AvZKeMLMXSh0ozZjApwEPE4p+TgMeknRKD9+Ac87VCCNn6aYUOu0BwczWmllLnL0OODixbkX8uwS4D3hnR4mlqQO4BHhXvs2/pHHAPcAvU+zrnHM1z8rX1fObPSAQTvxnUNAlv6SdzWxlnD2JOEKjpFHA5nhnMBY4AvhmR4mlyQAyBQ98raWPupBwzrn+xgza28tTKl6qBwRJVwALzGwO8HlJJwFtwDrgnLj73sD3JeUI5+irirQe2k6aDOAuSXOBm+L86fjTuc4596ZyVosW6wHBzC5NvL4YuLjIfn8G9u9KWmkqgS+QdDLhdgJglpnd2pVEnHOuVhm13RUEZvYr4FcVjsU556qPkfop3/6mZAYgaSMhc3vbKkLr0OEVi8o556pItTaML5kBmNmw3gzEOeeqVRmfBO5VfTUegHPO1QQzI1emVkC9rU8yAEmXA58EVsdFX4k134XbvQRsBNqBtjI8Qeecc2WX8iGvfqcv7wCuMbNvpdjuGDNbU/FonHOum6q1dxwvAnLOuR4wq946gL58ondm7M1udnyEuRgD7pa0sEiveduRdF6+h73Vq1d3tKlzzpVVF3oD7VcqlgFIukfSk0WmGcB/A7sTupdeCfx7icO8x8wOAk4APifpqFLpmdksM2s2s+Zx48aV++0451xRZkZ7ey7V1N9UrAjIzI5Ns52kHwB3lDhGvme7VyXdShgs4Q9lC9I558qgWh8E65MiIEk7J2Y/BDxZZJshkoblXwPTi23nnHN9rZxjAvemvqoE/qakaYQy/peATwFI2gW4zsxOBHYEbpWUj/NGM7urb8J1zrkSrMb7Aio3MzurxPK/ASfG10uAA3szLuec6yqjeouAvBmoc871iPlzAM45V5eMftnCJw3PAJxzrge8CMg55+pVLY4H4JxzLg3zzuCcc65e+R2Ac87VIcN7A3XOufpk0N7mrYCcc64O+XMAzjlXl8zActV5B9CX4wE451xNyOUs1ZSGpOMlPStpsaSLiqw/R9JqSY/G6dzEurMlPR+nsztLy+8AnHOuh8pVBCQpC1wLvB9YDsyXNMfMnirY9BYzm1mw72jgMqCZUDe9MO67vlR6fgfgnHM9YGbk2nKpphQOARab2RIz2wbcDMxIGcpxwDwzWxdP+vOA4zvaoa7vADZuaedv69rYui1HQ1bsMKKBMcOzZEIX1K7KWS7H2t8/yMs//AUtq1YzYJcdmPSJUxlzzLuRf8Y1Idfayrr77mP1b/+Xtk0bGThxIjufdipD9967d+Ow1HUAYyUtSMzPMrNZifnxwLLE/HLg0CLHOTmOkPgc8AUzW1Zi3/EdBVOXGUBbu7Ho5a1s3JLbrh/v195oJ7MS9p88kKGDsn0XoOuxLctW8tBx57B15au0b9r85vJXf/t7Bo7fiXfPvZ6B43fswwhdT2165hkWfXYmuZYWcpvDZ7zhr4+y5q65DN1/P/a+5ts0DBlS+UC61hXEGjNr7mGKvwFuMrMWSZ8Cfgy8tzsHqrsiIDPjiaVb2VBw8gdoz0FrOzz20la2bKvOWn0Hra9v5M9Hns7mJcu2O/kDtG/azObFS3ngyNNp27ipjyJ0PbVl2TKePPc82tavf/PkD0AuR27rVjY+9jhPfXZmr7TOMdKNBpYyk1gBTEzMT4jL3krPbK2ZtcTZ64CD0+5bqM8yAEnnS3pG0iJJ3yyxTYe14d2xflM7m1tydFRn056Dl1e3liM51weWzf4F29a9jrW3F11v7e1sW7ueZdf/upcjc+Wy7Ps/oH3LlpLrbds23nj+eV5/+OFeicfMUk0pzAemSpoiqQk4A5iT3KBgSN2TgKfj67nAdEmjJI0iDKM7t6PE+mpM4GMIFRsHmtm+wLeKbJOvDT8B2Ac4U9I+PU17+dpW0nTdvfr1NtqrtH+Pevfif1xPbsvWDrfJbd7Kku/8qJcicuXUvmULa++5Bzq5us9t2cKKn95Q+YAMcrlcqqnTQ5m1ATMJJ+6ngZ+b2SJJV0g6KW72+Xjh/BjweeCcuO864EpCJjIfuCIuK6mv6gA+A1yVv40xs1eLbPNmbTiApHxteGFzqC7Z0pL+pL6tzRjU5JWF1cTM2Lqy2Nfp7VpWvFLhaFwlbFu9BrLp6ui2vPhihaMJRUC5Eneb3Tqe2Z3AnQXLLk28vhi4uMS+s4HZadPqqyKgPYEjJT0k6X5J7yqyTZdqtCWdJ2mBpAWrV68umXBXGn9k/NxfdSShlCcHNXhFfzXKNDZAyhOumhorHA1vVgKXqQ6gV1UsA5B0j6Qni0wzCHceo4F3AxcAP1cP2+WZ2Swzazaz5nHjxpXcbszwLGkSamwQTQ2eA1Sj0UcWu554uzFHF2td5/q7pp12omHEiE63U2MjY445phci8gzgbczsWDPbr8h0O+Fq/tcWPAzkgLEFh+hyjXYa40c3dnoXkBFMGNPgbcWr1B4Xnkd2yKAOt8kOHsRu/3xuh9u4/kkS4885m8zAgR1vl8mw82mn9kJERs5yqab+pq+KgG4DjgGQtCfQBKwp2KbT2vDuGNiUYfedmkoW72QEIwZn2GV0L9w6uooY+77DmXDOKWQHF88EsoMHMfHc0xh79Lt7OTJXLjufegrDph1YMhPIDBjAlAsvYMDOOxddX05WxUVAfVUJPBuYLelJYBtwtpmZpF2A68zsRDNrk5SvDc8Cs81sUTkS33l0I02N4sVV29jaam/eEQgYP6aRSeMa/eq/yu17zSUM33cqz33tu7Rt2ISyWay9ncYRw5j6/2Yy8RO9cWXoKkUNDezzn//B8ut+yN9uuglyBhLW1saAXXZh1//7T4w+8j29Fk+19gaqau3HuiPNzc22YMGCzjcE3tiao6UtRzYjhg3KeDcQNcbMeH3hk2xbu54BY0cz/KB9PXOvMbnWVjYtWkT75s0M2HEnBu++W+p9JS3s6ZO5w0fvbe+a/uNU2957y6E9Tq+c6rIriKQhAzMMqb8HouuGJEY279/XYbgKyjQ2MnzatD5L3yB1V8/9Td1nAM451yNVPCCMZwDOOdcj/bOCNw3PAJxzroesHzbxTMMzAOec64EwIEz5uoLoTTXZCkjSamBpF3cby9ufRahl/n5rX7295+6838lmVrrrgBQk3cXbH2QtZY2ZdThKV2+qyQygOyQt6E/NsyrN32/tq7f3XG/vtxy8/aNzztUpzwCcc65OeQbwllmdb1JT/P3Wvnp7z/X2fnvM6wCcc65O+R2Ac87VKc8AnHOuTnkGUEDSlySZpLTtequWpCslPS7pUUl3x+64a5akqyU9E9/zrZJG9nVMlSbp1DiAeE5SzTaRlHS8pGclLZZ0UV/HUy08A0iQNBGYDrzc17H0kqvN7AAzmwbcAVza2Q5Vbh6wn5kdADxHiYG1a8yTwIeBP/R1IJUiKQtcC5wA7AOcKWmfvo2qOngGsL1rgAsJPbzWPDPbkJgdQo2/bzO728za4uyDhGFGa5qZPW1mz/Z1HBV2CLDYzJaY2TbgZmBGH8dUFbwvoCgOVr/CzB6rpwFDJH0d+DjwOnGYzjrxCeCWvg7ClcV4YFlifjlwaB/FUlXqKgOQdA+wU5FVlwBfIRT/1JSO3rOZ3W5mlwCXSLoYmAlc1qsBllln7zducwnQBtzQm7FVSpr37FwxdZUBmNmxxZZL2h+YAuSv/icAj0g6xMxW9WKIZVfqPRdxA3AnVZ4BdPZ+JZ0DfAB4n9XIQzBd+Ixr1QpgYmJ+QlzmOlFXGUApZvYEsEN+XtJLQLOZ1XRPipKmmtnzcXYG8ExfxlNpko4n1PH8nZlt7ut4XNnMB6ZKmkI48Z8BfKRvQ6oOngHUt6sk7QXkCN1nf7qP46m07wIDgHnxTu9BM6vp9yzpQ8B/AeOA30p61MyO6+OwysrM2iTNBOYCWWC2mS3q47CqgncF4ZxzdcqbgTrnXJ3yDMA55+qUZwDOOVenPANwzrk65RmAc87VKc8AXFlI2lSm41wv6ZRyHKuTdP5c6TQK0hsp6bO9maZznfEMwNUkSR0+42Jmh/dymiMBzwBcv+IZgCsrBVdLelLSE5JOj8szkr4X++OfJ+nOzq70JR0s6X5JCyXNlbRzXP5JSfMlPSbpV5IGx+XXS/ofSQ8B35R0uaTZku6TtETS5xPH3hT/Hh3X/zLGdoPiU2KSTozLFkr6T0l3FInxHElzJN0L/E7SUEm/k/RIfP/5XimvAnaPYy9cHfe9IL6PxyV9taf/e+e6zMx88qnHE7Ap/j2Z0O9+FtiRMLbCzsAphL6GMoSOy9YDpxQ5zvVx20bgz8C4uPx0whOeAGMS238NOD+x7x1ANs5fHo8xABgLrAUaC+I9mtAT6oQY21+A9wADCT1MTonb3QTcUSTecwi9T46O8w3A8Ph6LLAYELAr8GRiv+mEQcwV070DOKqvP0ef6mvyriBcub0HuMnM2oFXJN0PvCsu/4WZ5YBVkn7fyXH2AvbjrW4bssDKuG4/SV8jFKsMJXQBkPeLmHbeb82sBWiR9CohU1pekNbDZrYcQNKjhJP1JmCJmb0Yt7kJOK9ErPPMbF18LeAbko4idLExPqZZaHqc/hrnhwJTqeGBW1z/4xmA668ELDKzw4qsux74ewtjN5xDuIrPe6Ng25bE63aKf+fTbNORZJofJfS7c7CZtcaOBQcW2UfAv5rZ97uYlnNl43UArtz+CJwuKStpHHAU8DDwAHByrAvYke1P2sU8C4yTdBiApEZJ+8Z1w4CVkhoJJ9xKeBbYTdKucf70lPuNAF6NJ/9jgMlx+UZC3HlzgU9IGgogabykHXCuF/kdgCu3W4HDgMcIQ0xeaGarJP0KeB/wFKFs/RFC2XtRZrYtVhL/p6QRhO/qd4BFwP8DHgJWx7/DSh2nu8xsS2y2eZekNwhdDqdxA/AbSU8AC4hdbJvZWkkPSHoS+F8zu0DS3sBfYhHXJuBjwKvlfi/OleK9gbpeI2momW2SNIZwV3CE9eMBdxLxijDo+PNmdk1fx+VcufgdgOtNd0gaCTQBV/bnk3/0SUlnE+L9K+Dl9a6m+B2Ac87VKa8Eds65OuUZgHPO1SnPAJxzrk55BuCcc3XKMwDnnKtT/x8NeNU8a9m7cQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the cross-validation results\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "results = accuracies\n",
    "x_scatter = [math.log10(x[0]) for x in results]\n",
    "y_scatter = [math.log10(x[1]) for x in results]\n",
    "\n",
    "# plot validation accuracy\n",
    "marker_size=100\n",
    "colors = [results[x] for x in results] # default size of markers is 20\n",
    "\n",
    "plt.scatter(x_scatter, y_scatter, marker_size, c=colors, cmap=plt.cm.coolwarm)\n",
    "plt.colorbar()\n",
    "plt.xlabel('log learning rate')\n",
    "plt.ylabel('log regularization strength')\n",
    "plt.title('Net intrusion(CIC-IDS-2017) validation accuracy')\n",
    "plt.savefig(join(dataroot,'runs',method,'1st_run.png'))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}