cnn_pretrained.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named 'numpy'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-bbb04e9bfc42>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named 'numpy'"
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "import logging\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from torch.autograd import Variable\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "\n",
    "from sklearn.metrics import f1_score, recall_score, precision_score, roc_curve, roc_auc_score, auc\n",
    "\n",
    "import utils\n",
    "import models.net as net\n",
    "import models.data_loader as data_loader\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# This is a bit of magic to make matplotlib figures appear inline in the notebook\n",
    "# rather than in a new window.\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (30.0, 16.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "# Some more magic so that the notebook will reload external python modules;\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configure Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Directory containing the .txt metadata about dataset\n",
    "data_dir = 'data/4326'\n",
    "layer_name = 'VIIRS_SNPP_CorrectedReflectance_TrueColor'\n",
    "label_split = os.path.join(data_dir, layer_name + \".txt\")\n",
    "\n",
    "# Directory containing params.json\n",
    "model_dir = 'experiments/pretrained_model'\n",
    "\n",
    "# Name of the file in --model_dir containing weights to reload before training\n",
    "restore_file = 'best' # 'best' or 'last' or None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Function Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, loss_fn, dataloader, metrics, params):\n",
    "    \"\"\"Evaluate the model on `num_steps` batches.\n",
    "    Args:\n",
    "        model: (torch.nn.Module) the neural network\n",
    "        loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch\n",
    "        dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches data\n",
    "        metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch\n",
    "        params: (Params) hyperparameters\n",
    "    \"\"\"\n",
    "\n",
    "    # set model to evaluation mode\n",
    "    model.eval()\n",
    "\n",
    "    # summary for current eval loop\n",
    "    summ = []\n",
    "    \n",
    "    X = None\n",
    "    outputs_prob = None\n",
    "    outputs_pred = None\n",
    "    labels = None\n",
    "\n",
    "    # compute metrics over the dataset\n",
    "    with tqdm_notebook(total=len(dataloader)) as t:\n",
    "        for data_batch, labels_batch in dataloader:\n",
    "            # move to GPU if available\n",
    "            if params.cuda:\n",
    "                data_batch, labels_batch = data_batch.cuda(async=True), labels_batch.cuda(async=True)\n",
    "            # fetch the next evaluation batch\n",
    "            data_batch, labels_batch = Variable(data_batch), Variable(labels_batch)\n",
    "\n",
    "            # compute model output\n",
    "            output_batch = model(data_batch)\n",
    "            loss = loss_fn(output_batch, labels_batch)\n",
    "\n",
    "            # extract data from torch Variable, move to cpu, convert to numpy arrays\n",
    "            output_batch = output_batch.data.cpu().numpy()\n",
    "            output_pred_batch = np.argmax(output_batch, axis=1)\n",
    "            labels_batch = labels_batch.data.cpu().numpy()\n",
    "\n",
    "            # save the outputs and labels\n",
    "            if X is None:\n",
    "                X = data_batch\n",
    "                outputs_prob = output_batch\n",
    "                outputs_pred = output_pred_batch\n",
    "                labels = labels_batch\n",
    "            else:\n",
    "                X = np.append(X, data_batch, axis=0)\n",
    "                outputs_prob = np.append(outputs_prob, output_batch, axis=0)\n",
    "                outputs_pred = np.append(outputs_pred, output_pred_batch, axis=0)\n",
    "                labels = np.append(labels, labels_batch, axis=0)\n",
    "\n",
    "            # compute all metrics on this batch\n",
    "            summary_batch = {metric: metrics[metric](output_batch, labels_batch)\n",
    "                             for metric in metrics}\n",
    "            summary_batch['loss'] = loss.data[0]\n",
    "            summ.append(summary_batch)\n",
    "\n",
    "    # compute mean of all metrics in summary\n",
    "    metrics_mean = {metric:np.mean([x[metric] for x in summ]) for metric in summ[0]} \n",
    "    metrics_string = \" ; \".join(\"{}: {:05.3f}\".format(k, v) for k, v in metrics_mean.items())\n",
    "    logging.info(\"- Eval metrics : \" + metrics_string)\n",
    "    \n",
    "    return X, outputs_prob, outputs_pred, labels, metrics_mean\n",
    "\n",
    "def train(model, optimizer, loss_fn, dataloader, metrics, params):\n",
    "    \"\"\"Train the model on `num_steps` batches\n",
    "    Args:\n",
    "        model: (torch.nn.Module) the neural network\n",
    "        optimizer: (torch.optim) optimizer for parameters of model\n",
    "        loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch\n",
    "        dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches training data\n",
    "        metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch\n",
    "        params: (Params) hyperparameters\n",
    "    \"\"\"\n",
    "\n",
    "    # set model to training mode\n",
    "    model.train()\n",
    "\n",
    "    # summary for current training loop and a running average object for loss\n",
    "    summ = []\n",
    "    loss_avg = utils.RunningAverage()\n",
    "\n",
    "    # Use tqdm for progress bar\n",
    "    with tqdm_notebook(total=len(dataloader)) as t:\n",
    "        for i, (train_batch, labels_batch) in enumerate(dataloader):\n",
    "            # move to GPU if available\n",
    "            if params.cuda:\n",
    "                train_batch, labels_batch = train_batch.cuda(async=True), labels_batch.cuda(async=True)\n",
    "            # convert to torch Variables\n",
    "            train_batch, labels_batch = Variable(train_batch), Variable(labels_batch)\n",
    "\n",
    "            # compute model output and loss\n",
    "            output_batch = model(train_batch)\n",
    "            loss = loss_fn(output_batch, labels_batch)\n",
    "\n",
    "            # clear previous gradients, compute gradients of all variables wrt loss\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "\n",
    "            # performs updates using calculated gradients\n",
    "            optimizer.step()\n",
    "\n",
    "            # Evaluate summaries only once in a while\n",
    "            if i % params.save_summary_steps == 0:\n",
    "                # extract data from torch Variable, move to cpu, convert to numpy arrays\n",
    "                output_batch = output_batch.data.cpu().numpy()\n",
    "                labels_batch = labels_batch.data.cpu().numpy()\n",
    "\n",
    "                # compute all metrics on this batch\n",
    "                summary_batch = {metric:metrics[metric](output_batch, labels_batch)\n",
    "                                 for metric in metrics}\n",
    "                summary_batch['loss'] = loss.data[0]\n",
    "                summ.append(summary_batch)\n",
    "\n",
    "            # update the average loss\n",
    "            loss_avg.update(loss.data[0])\n",
    "\n",
    "            t.set_postfix(loss='{:05.3f}'.format(loss_avg()))\n",
    "            t.update()\n",
    "\n",
    "    # compute mean of all metrics in summary\n",
    "    metrics_mean = {metric:np.mean([x[metric] for x in summ]) for metric in summ[0]}\n",
    "    metrics_string = \" ; \".join(\"{}: {:05.3f}\".format(k, v) for k, v in metrics_mean.items())\n",
    "    logging.info(\"- Train metrics: \" + metrics_string)\n",
    "\n",
    "\n",
    "def train_and_evaluate(model, train_dataloader, val_dataloader, optimizer, loss_fn, metrics, params, model_dir,\n",
    "                       restore_file=None):\n",
    "    \"\"\"Train the model and evaluate every epoch.\n",
    "    Args:\n",
    "        model: (torch.nn.Module) the neural network\n",
    "        train_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches training data\n",
    "        val_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches validation data\n",
    "        optimizer: (torch.optim) optimizer for parameters of model\n",
    "        loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch\n",
    "        metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch\n",
    "        params: (Params) hyperparameters\n",
    "        model_dir: (string) directory containing config, weights and log\n",
    "        restore_file: (string) optional- name of file to restore from (without its extension .pth.tar)\n",
    "    \"\"\"\n",
    "    # reload weights from restore_file if specified\n",
    "    if restore_file is not None:\n",
    "        restore_path = os.path.join(model_dir, restore_file + '.pth.tar')\n",
    "        logging.info(\"Restoring parameters from {}\".format(restore_path))\n",
    "        utils.load_checkpoint(restore_path, model, optimizer)\n",
    "\n",
    "    best_val_score = 0.0\n",
    "\n",
    "    for epoch in range(params.num_epochs):\n",
    "        # Run one epoch\n",
    "        logging.info(\"Epoch {}/{}\".format(epoch + 1, params.num_epochs))\n",
    "\n",
    "        # compute number of batches in one epoch (one full pass over the training set)\n",
    "        train(model, optimizer, loss_fn, train_dataloader, metrics, params)\n",
    "\n",
    "        # Evaluate for one epoch on validation set\n",
    "        _, _, _, _, val_metrics = evaluate(model, loss_fn, val_dataloader, metrics, params)\n",
    "\n",
    "        val_score = val_metrics['AUC']\n",
    "        is_best = val_score>best_val_score\n",
    "\n",
    "        # Save weights\n",
    "        utils.save_checkpoint({'epoch': epoch + 1,\n",
    "                               'state_dict': model.state_dict(),\n",
    "                               'optim_dict' : optimizer.state_dict()},\n",
    "                               is_best=is_best,\n",
    "                               checkpoint=model_dir)\n",
    "\n",
    "        # If best_eval, best_save_path\n",
    "        if is_best:\n",
    "            logging.info(\"- Found new best accuracy\")\n",
    "            print(\"best_val_score {:05.3f}\".format(best_val_score))\n",
    "            print(\"val_score {:05.3f}\".format(val_score))\n",
    "            best_val_score = val_score\n",
    "\n",
    "            # Save best val metrics in a json file in the model directory\n",
    "            best_json_path = os.path.join(model_dir, \"metrics_val_best_weights.json\")\n",
    "            utils.save_dict_to_json(val_metrics, best_json_path)\n",
    "\n",
    "        # Save latest val metrics in a json file in the model directory\n",
    "        last_json_path = os.path.join(model_dir, \"metrics_val_last_weights.json\")\n",
    "        utils.save_dict_to_json(val_metrics, last_json_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Hyperparameters from File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU available: True\n"
     ]
    }
   ],
   "source": [
    "json_path = os.path.join(model_dir, 'params.json')\n",
    "assert os.path.isfile(json_path), \"No json configuration file found at {}\".format(json_path)\n",
    "params = utils.Params(json_path)\n",
    "\n",
    "# use GPU if available\n",
    "params.cuda = torch.cuda.is_available()\n",
    "print(\"GPU available: {}\".format(params.cuda))\n",
    "\n",
    "# Set the random seed for reproducible experiments\n",
    "torch.manual_seed(230)\n",
    "if params.cuda: torch.cuda.manual_seed(230)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the Model, Loss, and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\xue\\miniconda3\\lib\\site-packages\\torchvision\\models\\densenet.py:212: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  nn.init.kaiming_normal(m.weight.data)\n"
     ]
    }
   ],
   "source": [
    "# Define the model and optimizer\n",
    "model = models.densenet121(pretrained='imagenet').cuda() if params.cuda else models.densenet121(pretrained='imagenet')\n",
    "\n",
    "# Freeze all the layers!\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "# Change the last layer to our number of classes\n",
    "# Parameters of newly constructed modules have requires_grad=True by default\n",
    "n_features = model.classifier.in_features\n",
    "n_class = 2\n",
    "model.classifier = nn.Linear(n_features, n_class).cuda() if params.cuda else nn.Linear(n_features, n_class)\n",
    "\n",
    "# Unfreeze the last layer\n",
    "for param in model.classifier.parameters():\n",
    "    param.requires_grad = True\n",
    "\n",
    "# Define the loss\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "# Optimize only the classifier\n",
    "optimizer = optim.Adam(model.classifier.parameters(), lr=params.learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch metrics\n",
    "def accuracy(outputs, labels):\n",
    "    \"\"\"\n",
    "    Compute the accuracy, given the outputs and labels for all images.\n",
    "\n",
    "    Args:\n",
    "        outputs: (np.ndarray) dimension batch_size x 2 - log softmax output of the model\n",
    "        labels: (np.ndarray) dimension batch_size, where each element is a value in [0, 1]\n",
    "\n",
    "    Returns: (float) accuracy in [0,1]\n",
    "    \"\"\"\n",
    "    outputs = np.argmax(outputs, axis=1)\n",
    "    return np.sum(outputs==labels)/float(labels.size)\n",
    "\n",
    "def recall(outputs, labels):\n",
    "    \"\"\"\n",
    "    Compute the recall, given the outputs and labels for all images.\n",
    "\n",
    "    Args:\n",
    "        outputs: (np.ndarray) dimension batch_size x 2 - log softmax output of the model\n",
    "        labels: (np.ndarray) dimension batch_size, where each element is a value in [0, 1]\n",
    "\n",
    "    Returns: (float) recall in [0,1]\n",
    "    \"\"\"\n",
    "    outputs = np.argmax(outputs, axis=1)\n",
    "    return recall_score(labels, outputs)\n",
    "\n",
    "def precision(outputs, labels):\n",
    "    \"\"\"\n",
    "    Compute the precision, given the outputs and labels for all images. \n",
    "    \n",
    "    The precision is the ratio tp / (tp + fp) where tp is the number \n",
    "    of true positives and fp the number of false positives. \n",
    "    \n",
    "    The precision is intuitively the ability of the classifier not to \n",
    "    label as positive a sample that is negative.\n",
    "    \n",
    "    The best value is 1 and the worst value is 0.\n",
    "    \n",
    "    Args:\n",
    "        outputs: (np.ndarray) dimension batch_size x 2 - log softmax output of the model\n",
    "        labels: (np.ndarray) dimension batch_size, where each element is a value in [0, 1]\n",
    "\n",
    "    Returns: (float) precision in [0,1]\n",
    "    \"\"\"\n",
    "    outputs = np.argmax(outputs, axis=1)\n",
    "    return precision_score(labels, outputs)\n",
    "\n",
    "def f1_metric(outputs, labels):\n",
    "    \"\"\"\n",
    "    Compute the F1 score, given the outputs and labels for all images.\n",
    "\n",
    "    Args:\n",
    "        outputs: (np.ndarray) dimension batch_size x 2 - log softmax output of the model\n",
    "        labels: (np.ndarray) dimension batch_size, where each element is a value in [0, 1]\n",
    "\n",
    "    Returns: (float) F1 score in [0,1]\n",
    "    \"\"\"\n",
    "    outputs = np.argmax(outputs, axis=1)\n",
    "    return f1_score(labels, outputs)\n",
    "\n",
    "def roc_graph(outputs, labels):\n",
    "    \"\"\"\n",
    "    Compute Receiver operating characteristic (ROC).\n",
    "    \n",
    "    Args:\n",
    "        outputs: (np.ndarray) dimension batch_size x 2 - log softmax output of the model\n",
    "        labels: (np.ndarray) dimension batch_size, where each element is a value in [0, 1]\n",
    "        \n",
    "    Returns: fpr, tpr, thresholds\n",
    "    \"\"\"\n",
    "    outputs = outputs[:,1]\n",
    "    return roc_curve(labels, outputs)\n",
    "\n",
    "def roc_auc_metric(outputs, labels):\n",
    "    \"\"\"\n",
    "    Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores.\n",
    "    \"\"\"\n",
    "    outputs = outputs[:,1]\n",
    "    return roc_auc_score(labels, outputs)\n",
    "\n",
    "# maintain all metrics required in this dictionary- these are used in the training and evaluation loops\n",
    "metrics = {\n",
    "    'AUC': roc_auc_metric, # area under receiver operator characteristic curve\n",
    "    'F1': f1_metric, # weighted average of precision and recall\n",
    "    'recall': recall, \n",
    "    'precision': precision,\n",
    "    'accuracy': accuracy,\n",
    "    # could add more metrics such as accuracy for each token type\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading the datasets...\n",
      "- done.\n"
     ]
    }
   ],
   "source": [
    "# Set the logger\n",
    "utils.set_logger(os.path.join(model_dir, 'train.log'))\n",
    "\n",
    "# Create the input data pipeline\n",
    "logging.info(\"Loading the datasets...\")\n",
    "\n",
    "# Set the model input size\n",
    "IMG_DIM = (112, 224)\n",
    "IMG_PADDING = (0, 56, 0, 56) # left, top, right, bottom borders\n",
    "IMG_BINZARIZE = False # use colored images\n",
    "\n",
    "# Apply normalization for Densenet pretrained model\n",
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                std=[0.229, 0.224, 0.225])\n",
    "\n",
    "# borrowed from http://pytorch.org/tutorials/advanced/neural_style_tutorial.html\n",
    "# and http://pytorch.org/tutorials/beginner/data_loading_tutorial.html\n",
    "# define a training image loader that specifies transforms on images. See documentation for more details.\n",
    "train_transformer = transforms.Compose([\n",
    "    transforms.Resize(IMG_DIM),  # resize the image \n",
    "    transforms.Pad(padding=IMG_PADDING, fill=0), # pad to be square!\n",
    "    transforms.RandomHorizontalFlip(),  # randomly flip image horizontally\n",
    "    transforms.RandomVerticalFlip(), # randomly flip image vertically\n",
    "    transforms.ToTensor(), # transform it into a torch tensor\n",
    "    normalize,\n",
    "    ])  \n",
    "\n",
    "# loader for evaluation, no data augmentation (e.g. horizontal flip)\n",
    "eval_transformer = transforms.Compose([\n",
    "    transforms.Resize(IMG_DIM),  # resize the image\n",
    "    transforms.Pad(padding=IMG_PADDING, fill=0), # pad to be square!\n",
    "    transforms.ToTensor(), # transform it into a torch tensor\n",
    "    normalize,\n",
    "    ])  \n",
    "\n",
    "params.img_binarize = IMG_BINZARIZE\n",
    "\n",
    "# fetch dataloaders\n",
    "train_dl = data_loader.fetch_dataloader('train', train_transformer, data_dir, layer_name, label_split, params)\n",
    "val_dl = data_loader.fetch_dataloader('val', eval_transformer, data_dir, layer_name, label_split, params)\n",
    "test_dl = data_loader.fetch_dataloader('test', eval_transformer, data_dir, layer_name, label_split, params)\n",
    "\n",
    "logging.info(\"- done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Starting training for 100 epoch(s)\n",
      "Restoring parameters from experiments/pretrained_model\\best.pth.tar\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\xue\\miniconda3\\lib\\site-packages\\ipykernel_launcher.py:112: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "c:\\users\\xue\\miniconda3\\lib\\site-packages\\ipykernel_launcher.py:116: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.864 ; F1: 0.182 ; recall: 0.100 ; precision: 1.000 ; accuracy: 0.859 ; loss: 0.307\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\xue\\miniconda3\\lib\\site-packages\\ipykernel_launcher.py:55: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "c:\\users\\xue\\miniconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "c:\\users\\xue\\miniconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.537 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.268\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.000\n",
      "val_score 0.537\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.853 ; F1: 0.316 ; recall: 0.188 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.277\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.539 ; F1: 0.143 ; recall: 0.094 ; precision: 0.300 ; accuracy: 0.925 ; loss: 0.266\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.537\n",
      "val_score 0.539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.762 ; F1: 0.333 ; recall: 0.214 ; precision: 0.750 ; accuracy: 0.906 ; loss: 0.273\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.529 ; F1: 0.143 ; recall: 0.094 ; precision: 0.300 ; accuracy: 0.925 ; loss: 0.266\n",
      "Epoch 4/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.831 ; F1: 0.421 ; recall: 0.267 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.256\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.530 ; F1: 0.143 ; recall: 0.094 ; precision: 0.300 ; accuracy: 0.925 ; loss: 0.264\n",
      "Epoch 5/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.859 ; F1: 0.222 ; recall: 0.125 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.251\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.143 ; recall: 0.094 ; precision: 0.300 ; accuracy: 0.925 ; loss: 0.263\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.539\n",
      "val_score 0.543\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.870 ; F1: 0.300 ; recall: 0.188 ; precision: 0.750 ; accuracy: 0.891 ; loss: 0.273\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.548 ; F1: 0.143 ; recall: 0.094 ; precision: 0.300 ; accuracy: 0.925 ; loss: 0.263\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.543\n",
      "val_score 0.548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.797 ; F1: 0.286 ; recall: 0.176 ; precision: 0.750 ; accuracy: 0.883 ; loss: 0.305\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.263\n",
      "Epoch 8/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.875 ; F1: 0.381 ; recall: 0.235 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.248\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.262\n",
      "Epoch 9/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.869 ; F1: 0.143 ; recall: 0.077 ; precision: 1.000 ; accuracy: 0.906 ; loss: 0.237\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.263\n",
      "Epoch 10/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.915 ; F1: 0.444 ; recall: 0.286 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.221\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.539 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.264\n",
      "Epoch 11/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.773 ; F1: 0.364 ; recall: 0.222 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.300\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.541 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.264\n",
      "Epoch 12/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.821 ; F1: 0.286 ; recall: 0.167 ; precision: 1.000 ; accuracy: 0.883 ; loss: 0.300\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.544 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.265\n",
      "Epoch 13/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.876 ; F1: 0.400 ; recall: 0.250 ; precision: 1.000 ; accuracy: 0.930 ; loss: 0.218\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.263\n",
      "Epoch 14/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.859 ; F1: 0.435 ; recall: 0.278 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.285\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.552 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.261\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.548\n",
      "val_score 0.552\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.824 ; F1: 0.300 ; recall: 0.176 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.278\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.552 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.262\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.552\n",
      "val_score 0.552\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.916 ; F1: 0.476 ; recall: 0.312 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.551 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.261\n",
      "Epoch 17/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.739 ; F1: 0.286 ; recall: 0.167 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.246\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.553 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.552\n",
      "val_score 0.553\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.892 ; F1: 0.381 ; recall: 0.235 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.262\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.556 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.553\n",
      "val_score 0.556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.783 ; F1: 0.333 ; recall: 0.200 ; precision: 1.000 ; accuracy: 0.844 ; loss: 0.382\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.261\n",
      "Epoch 20/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.755 ; F1: 0.200 ; recall: 0.118 ; precision: 0.667 ; accuracy: 0.875 ; loss: 0.328\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.259\n",
      "Epoch 21/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.817 ; F1: 0.357 ; recall: 0.217 ; precision: 1.000 ; accuracy: 0.859 ; loss: 0.350\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.261\n",
      "Epoch 22/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.844 ; F1: 0.250 ; recall: 0.143 ; precision: 1.000 ; accuracy: 0.906 ; loss: 0.248\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.551 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.259\n",
      "Epoch 23/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.962 ; F1: 0.526 ; recall: 0.357 ; precision: 1.000 ; accuracy: 0.930 ; loss: 0.204\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.555 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.259\n",
      "Epoch 24/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.874 ; F1: 0.455 ; recall: 0.294 ; precision: 1.000 ; accuracy: 0.906 ; loss: 0.263\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.556 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.261\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.556\n",
      "val_score 0.556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.882 ; F1: 0.471 ; recall: 0.308 ; precision: 1.000 ; accuracy: 0.930 ; loss: 0.225\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.551 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.262\n",
      "Epoch 26/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.864 ; F1: 0.462 ; recall: 0.300 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.280\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.554 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.262\n",
      "Epoch 27/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.883 ; F1: 0.421 ; recall: 0.267 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.554 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "Epoch 28/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.812 ; F1: 0.381 ; recall: 0.235 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.286\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.545 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.261\n",
      "Epoch 29/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.871 ; F1: 0.273 ; recall: 0.158 ; precision: 1.000 ; accuracy: 0.875 ; loss: 0.284\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "Epoch 30/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.869 ; F1: 0.182 ; recall: 0.100 ; precision: 1.000 ; accuracy: 0.859 ; loss: 0.309\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.542 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.259\n",
      "Epoch 31/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.889 ; F1: 0.182 ; recall: 0.105 ; precision: 0.667 ; accuracy: 0.859 ; loss: 0.286\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.542 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.257\n",
      "Epoch 32/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.933 ; F1: 0.118 ; recall: 0.067 ; precision: 0.500 ; accuracy: 0.883 ; loss: 0.232\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.542 ; F1: 0.136 ; recall: 0.094 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.257\n",
      "Epoch 33/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.759 ; F1: 0.095 ; recall: 0.053 ; precision: 0.500 ; accuracy: 0.852 ; loss: 0.336\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.259\n",
      "Epoch 34/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.856 ; F1: 0.364 ; recall: 0.222 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.270\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.550 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.258\n",
      "Epoch 35/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.838 ; F1: 0.370 ; recall: 0.227 ; precision: 1.000 ; accuracy: 0.867 ; loss: 0.339\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.259\n",
      "Epoch 36/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.805 ; F1: 0.200 ; recall: 0.118 ; precision: 0.667 ; accuracy: 0.875 ; loss: 0.277\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.258\n",
      "Epoch 37/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.892 ; F1: 0.444 ; recall: 0.286 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.223\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.551 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.260\n",
      "Epoch 38/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.943 ; F1: 0.471 ; recall: 0.333 ; precision: 0.800 ; accuracy: 0.930 ; loss: 0.198\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.550 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.259\n",
      "Epoch 39/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.870 ; F1: 0.400 ; recall: 0.250 ; precision: 1.000 ; accuracy: 0.883 ; loss: 0.286\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.258\n",
      "Epoch 40/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.783 ; F1: 0.250 ; recall: 0.154 ; precision: 0.667 ; accuracy: 0.906 ; loss: 0.267\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.538 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.258\n",
      "Epoch 41/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.856 ; F1: 0.462 ; recall: 0.300 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.282\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.535 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.257\n",
      "Epoch 42/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.888 ; F1: 0.348 ; recall: 0.211 ; precision: 1.000 ; accuracy: 0.883 ; loss: 0.279\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.539 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.257\n",
      "Epoch 43/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.835 ; F1: 0.267 ; recall: 0.154 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.234\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.535 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.256\n",
      "Epoch 44/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.861 ; F1: 0.118 ; recall: 0.062 ; precision: 1.000 ; accuracy: 0.883 ; loss: 0.259\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.548 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.253\n",
      "Epoch 45/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.871 ; F1: 0.571 ; recall: 0.400 ; precision: 1.000 ; accuracy: 0.930 ; loss: 0.242\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.555 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.256\n",
      "Epoch 46/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.778 ; F1: 0.444 ; recall: 0.286 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.240\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.548 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.256\n",
      "Epoch 47/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.873 ; F1: 0.421 ; recall: 0.267 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.247\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.540 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.257\n",
      "Epoch 48/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.869 ; F1: 0.316 ; recall: 0.188 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.274\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "Epoch 49/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.921 ; F1: 0.667 ; recall: 0.500 ; precision: 1.000 ; accuracy: 0.961 ; loss: 0.185\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "Epoch 50/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.898 ; F1: 0.462 ; recall: 0.333 ; precision: 0.750 ; accuracy: 0.945 ; loss: 0.201\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.539 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "Epoch 51/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.901 ; F1: 0.500 ; recall: 0.333 ; precision: 1.000 ; accuracy: 0.875 ; loss: 0.300\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.538 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.260\n",
      "Epoch 52/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.845 ; F1: 0.417 ; recall: 0.263 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.280\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.539 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.257\n",
      "Epoch 53/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.961 ; F1: 0.545 ; recall: 0.375 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.210\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.557 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.253\n",
      "- Found new best accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_score 0.556\n",
      "val_score 0.557\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 54/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.922 ; F1: 0.636 ; recall: 0.467 ; precision: 1.000 ; accuracy: 0.938 ; loss: 0.212\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.544 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.253\n",
      "Epoch 55/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.895 ; F1: 0.429 ; recall: 0.286 ; precision: 0.857 ; accuracy: 0.875 ; loss: 0.280\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.541 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.253\n",
      "Epoch 56/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.936 ; F1: 0.583 ; recall: 0.412 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.219\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.545 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.255\n",
      "Epoch 57/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.996 ; F1: 0.842 ; recall: 0.727 ; precision: 1.000 ; accuracy: 0.977 ; loss: 0.146\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.541 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.254\n",
      "Epoch 58/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.898 ; F1: 0.444 ; recall: 0.300 ; precision: 0.857 ; accuracy: 0.883 ; loss: 0.292\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.544 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.256\n",
      "Epoch 59/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.883 ; F1: 0.500 ; recall: 0.333 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.234\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.544 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.256\n",
      "Epoch 60/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.953 ; F1: 0.690 ; recall: 0.526 ; precision: 1.000 ; accuracy: 0.930 ; loss: 0.217\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.545 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.256\n",
      "Epoch 61/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.905 ; F1: 0.500 ; recall: 0.333 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.256\n",
      "Epoch 62/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.915 ; F1: 0.750 ; recall: 0.600 ; precision: 1.000 ; accuracy: 0.969 ; loss: 0.172\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.256\n",
      "Epoch 63/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.831 ; F1: 0.457 ; recall: 0.296 ; precision: 1.000 ; accuracy: 0.852 ; loss: 0.376\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.256\n",
      "Epoch 64/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.885 ; F1: 0.375 ; recall: 0.231 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.233\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.253\n",
      "Epoch 65/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.927 ; F1: 0.480 ; recall: 0.316 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.234\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.254\n",
      "Epoch 66/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.808 ; F1: 0.545 ; recall: 0.375 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.252\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.551 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.252\n",
      "Epoch 67/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.929 ; F1: 0.556 ; recall: 0.385 ; precision: 1.000 ; accuracy: 0.938 ; loss: 0.197\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.253\n",
      "Epoch 68/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.945 ; F1: 0.560 ; recall: 0.389 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.253\n",
      "Epoch 69/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.891 ; F1: 0.471 ; recall: 0.308 ; precision: 1.000 ; accuracy: 0.930 ; loss: 0.222\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.548 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.254\n",
      "Epoch 70/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.867 ; F1: 0.400 ; recall: 0.250 ; precision: 1.000 ; accuracy: 0.930 ; loss: 0.194\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.255\n",
      "Epoch 71/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.909 ; F1: 0.583 ; recall: 0.412 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.221\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.257\n",
      "Epoch 72/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.868 ; F1: 0.267 ; recall: 0.154 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.241\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.254\n",
      "Epoch 73/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.903 ; F1: 0.538 ; recall: 0.368 ; precision: 1.000 ; accuracy: 0.906 ; loss: 0.250\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.253\n",
      "Epoch 74/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.901 ; F1: 0.286 ; recall: 0.182 ; precision: 0.667 ; accuracy: 0.922 ; loss: 0.221\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.253\n",
      "Epoch 75/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.884 ; F1: 0.583 ; recall: 0.438 ; precision: 0.875 ; accuracy: 0.922 ; loss: 0.233\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.251\n",
      "Epoch 76/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.857 ; F1: 0.421 ; recall: 0.267 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.251\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.548 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.250\n",
      "Epoch 77/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.875 ; F1: 0.500 ; recall: 0.333 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.225\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.252\n",
      "Epoch 78/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.809 ; F1: 0.267 ; recall: 0.167 ; precision: 0.667 ; accuracy: 0.914 ; loss: 0.231\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.253\n",
      "Epoch 79/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.922 ; F1: 0.609 ; recall: 0.467 ; precision: 0.875 ; accuracy: 0.930 ; loss: 0.213\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.253\n",
      "Epoch 80/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.843 ; F1: 0.417 ; recall: 0.263 ; precision: 1.000 ; accuracy: 0.891 ; loss: 0.277\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.548 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.253\n",
      "Epoch 81/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.850 ; F1: 0.476 ; recall: 0.333 ; precision: 0.833 ; accuracy: 0.914 ; loss: 0.247\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.539 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.251\n",
      "Epoch 82/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.950 ; F1: 0.500 ; recall: 0.333 ; precision: 1.000 ; accuracy: 0.906 ; loss: 0.251\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.552 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.248\n",
      "Epoch 83/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.862 ; F1: 0.522 ; recall: 0.353 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.235\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.545 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.248\n",
      "Epoch 84/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.861 ; F1: 0.632 ; recall: 0.462 ; precision: 1.000 ; accuracy: 0.945 ; loss: 0.209\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.544 ; F1: 0.174 ; recall: 0.125 ; precision: 0.286 ; accuracy: 0.925 ; loss: 0.248\n",
      "Epoch 85/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.934 ; F1: 0.400 ; recall: 0.250 ; precision: 1.000 ; accuracy: 0.883 ; loss: 0.257\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.167 ; recall: 0.125 ; precision: 0.250 ; accuracy: 0.921 ; loss: 0.249\n",
      "Epoch 86/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.937 ; F1: 0.538 ; recall: 0.368 ; precision: 1.000 ; accuracy: 0.906 ; loss: 0.237\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.543 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.251\n",
      "Epoch 87/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.904 ; F1: 0.621 ; recall: 0.450 ; precision: 1.000 ; accuracy: 0.914 ; loss: 0.249\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.254\n",
      "Epoch 88/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.876 ; F1: 0.552 ; recall: 0.381 ; precision: 1.000 ; accuracy: 0.898 ; loss: 0.286\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.544 ; F1: 0.231 ; recall: 0.188 ; precision: 0.300 ; accuracy: 0.929 ; loss: 0.253\n",
      "Epoch 89/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.914 ; F1: 0.444 ; recall: 0.286 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.206\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.549 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.251\n",
      "Epoch 90/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.811 ; F1: 0.316 ; recall: 0.200 ; precision: 0.750 ; accuracy: 0.898 ; loss: 0.267\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.550 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.252\n",
      "Epoch 91/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.876 ; F1: 0.632 ; recall: 0.462 ; precision: 1.000 ; accuracy: 0.945 ; loss: 0.200\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.551 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.251\n",
      "Epoch 92/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.934 ; F1: 0.636 ; recall: 0.500 ; precision: 0.875 ; accuracy: 0.938 ; loss: 0.193\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.550 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.251\n",
      "Epoch 93/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.905 ; F1: 0.545 ; recall: 0.375 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.225\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.550 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.250\n",
      "Epoch 94/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.913 ; F1: 0.696 ; recall: 0.533 ; precision: 1.000 ; accuracy: 0.945 ; loss: 0.185\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.549 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.251\n",
      "Epoch 95/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.886 ; F1: 0.500 ; recall: 0.333 ; precision: 1.000 ; accuracy: 0.875 ; loss: 0.302\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.542 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.249\n",
      "Epoch 96/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.912 ; F1: 0.538 ; recall: 0.368 ; precision: 1.000 ; accuracy: 0.906 ; loss: 0.255\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.550 ; F1: 0.200 ; recall: 0.156 ; precision: 0.278 ; accuracy: 0.925 ; loss: 0.248\n",
      "Epoch 97/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.923 ; F1: 0.533 ; recall: 0.400 ; precision: 0.800 ; accuracy: 0.891 ; loss: 0.267\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.545 ; F1: 0.231 ; recall: 0.188 ; precision: 0.300 ; accuracy: 0.929 ; loss: 0.250\n",
      "Epoch 98/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.975 ; F1: 0.583 ; recall: 0.438 ; precision: 0.875 ; accuracy: 0.922 ; loss: 0.182\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.546 ; F1: 0.231 ; recall: 0.188 ; precision: 0.300 ; accuracy: 0.929 ; loss: 0.248\n",
      "Epoch 99/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.740 ; F1: 0.462 ; recall: 0.300 ; precision: 1.000 ; accuracy: 0.945 ; loss: 0.225\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.545 ; F1: 0.231 ; recall: 0.188 ; precision: 0.300 ; accuracy: 0.929 ; loss: 0.249\n",
      "Epoch 100/100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Train metrics: AUC: 0.938 ; F1: 0.545 ; recall: 0.375 ; precision: 1.000 ; accuracy: 0.922 ; loss: 0.213\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "- Eval metrics : AUC: 0.547 ; F1: 0.231 ; recall: 0.188 ; precision: 0.300 ; accuracy: 0.929 ; loss: 0.248\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "logging.info(\"Starting training for {} epoch(s)\".format(params.num_epochs))\n",
    "train_and_evaluate(model, train_dl, val_dl, optimizer, loss_fn, metrics, params, model_dir, restore_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate Model on Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Starting evaluation\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xue/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:55: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "- Eval metrics : AUC: 0.715 ; F1: 0.552 ; recall: 0.479 ; precision: 0.852 ; accuracy: 0.922 ; loss: 0.301\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Get the logger\n",
    "utils.set_logger(os.path.join(model_dir, 'evaluate.log'))\n",
    "\n",
    "logging.info(\"Starting evaluation\")\n",
    "\n",
    "# Reload weights from the saved file\n",
    "restore_file = 'best'\n",
    "utils.load_checkpoint(os.path.join(model_dir, restore_file + '.pth.tar'), model)\n",
    "\n",
    "# Evaluate\n",
    "X_test, y_test_prob, y_test_pred, y_test, test_metrics = evaluate(model, loss_fn, test_dl, metrics, params)\n",
    "save_path = os.path.join(model_dir, \"metrics_test_{}.json\".format(restore_file))\n",
    "utils.save_dict_to_json(test_metrics, save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_fpr, test_tpr, test_thresholds = roc_graph(y_test_prob, y_test)\n",
    "test_roc_auc = auc(test_fpr, test_tpr)\n",
    "\n",
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(test_fpr, test_tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % test_roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic for test set')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confusion_matrix(results, categories, normalizeMatrix = True, plotConfusion=False):\n",
    "    \"\"\" Generate confusion matrix\n",
    "    - results = list of tuples of (correct label, predicted label)\n",
    "    - categories = list of category names\n",
    "    Returns confusion matrix; rows are correct labels and columns are predictions\n",
    "    \"\"\"\n",
    "    # Empty confusion matrix\n",
    "    matrix = np.zeros((len(categories),len(categories)))\n",
    "\n",
    "    # Iterate over all labels and populate matrix\n",
    "    for label, pred in results:\n",
    "        matrix[label, pred] += 1\n",
    "\n",
    "    # Print matrix and percent accuracy\n",
    "    accuracy = float(np.trace(matrix)) * 100.0 / len(results)\n",
    "    print('Accuracy: ' +  str(accuracy) + '%')\n",
    "\n",
    "    # Normalize confusion matrix\n",
    "    if normalizeMatrix:\n",
    "      print(\"Non-normalized conf_matrix:\")\n",
    "      print(matrix)\n",
    "      s = np.sum(matrix,1) # Sum each row\n",
    "      for i in range(matrix.shape[0]):\n",
    "        # Normalization handles class imbalance in training set\n",
    "        matrix[i,:] /= s[i]\n",
    "\n",
    "    # Save matrix to file:\n",
    "    # np.save(\"confusion_matrix.npy\",matrix)\n",
    "\n",
    "    # Plot the confusion matrix\n",
    "    if plotConfusion:\n",
    "        plot_confusion_matrix(matrix, categories)\n",
    "\n",
    "def plot_confusion_matrix(cm, names=None, title='Confusion Matrix', cmap=plt.cm.Blues):\n",
    "    plt.figure(4)\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "\n",
    "    # Add labels to confusion matrix:\n",
    "    if names is None:\n",
    "        names = range(cm.shape[0])\n",
    "\n",
    "    tick_marks = np.arange(len(names))\n",
    "    plt.xticks(tick_marks, names, rotation=45)\n",
    "    plt.yticks(tick_marks, names)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('Correct label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "\n",
    "classes = ['normal', 'abnormal']\n",
    "confusion_matrix(list(zip(y_test_pred, y_test)), classes, plotConfusion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize misclassified examples from the validation dataset.\n",
    "num_classes = len(classes)\n",
    "max_examples = 5\n",
    "\n",
    "incorrect_idx = [idx for idx, incorrect in enumerate(np.array(y_test) != np.array(y_test_pred)) if incorrect]\n",
    "if len(incorrect_idx) > max_examples:\n",
    "    incorrect_idx = incorrect_idx[:max_examples]\n",
    "num_incorrect = len(incorrect_idx)\n",
    "\n",
    "for y, idx in enumerate(incorrect_idx):\n",
    "    for i in range(3):\n",
    "        plt_idx = i * num_incorrect + y + 1\n",
    "        if i == 0:\n",
    "            plt.subplot(3, num_incorrect, plt_idx)            \n",
    "            plt.imshow(np.transpose(X_test[idx]*255., (1, 2, 0)).astype('uint8'))\n",
    "            plt.axis('off')\n",
    "            plt.title(\"Prediction: %s\\n Truth: %s\" % (classes[y_test_pred[idx]], classes[y_test[idx]]))\n",
    "        elif i == 1:\n",
    "            plt.subplot(3, num_incorrect, plt_idx)\n",
    "            log_probabilities = y_test_prob[idx]\n",
    "            probabilities = np.exp(log_probabilities)\n",
    "            y_pos = np.arange(len(classes))\n",
    "            plt.bar(y_pos, probabilities)\n",
    "            plt.xticks(y_pos, classes, rotation=45)\n",
    "            plt.ylabel(\"Probability\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num_classes = len(classes)\n",
    "# # img_size = (16, 8)\n",
    "# # num_channels = 1\n",
    "\n",
    "# # Visualize the learned weights for each class.\n",
    "# finalconv_name = 'features'\n",
    "\n",
    "# # get the softmax weight\n",
    "# params = list(net.parameters())\n",
    "# weight_softmax = np.squeeze(params[-2].data.numpy())\n",
    "\n",
    "# def returnCAM(feature_conv, weight_softmax, class_idx):\n",
    "#     # generate the class activation maps upsample to 256x256\n",
    "#     size_upsample = (256, 256)\n",
    "#     bz, nc, h, w = feature_conv.shape\n",
    "#     output_cam = []\n",
    "#     for idx in class_idx:\n",
    "#         cam = weight_softmax[idx].dot(feature_conv.reshape((nc, h*w)))\n",
    "#         cam = cam.reshape(h, w)\n",
    "#         cam = cam - np.min(cam)\n",
    "#         cam_img = cam / np.max(cam)\n",
    "#         cam_img = np.uint8(255 * cam_img)\n",
    "#         output_cam.append(cv2.resize(cam_img, size_upsample))\n",
    "#     return output_cam\n",
    "\n",
    "# # hook the feature extractor\n",
    "# features_blobs = []\n",
    "# def hook_feature(module, input, output):\n",
    "#     features_blobs.append(output.data.cpu().numpy())\n",
    "\n",
    "# model._modules.get(finalconv_name).register_forward_hook(hook_feature)\n",
    "\n",
    "\n",
    "# # w = model.fc2.weight.data.numpy() # convert to numpy arrays\n",
    "# # print(w.shape)\n",
    "# # w = w.reshape(num_classes, img_size[1], img_size[0], num_channels)\n",
    "# # w_min, w_max = np.min(w), np.max(w)\n",
    "# # for i in range(num_classes):\n",
    "# #     plt.subplot(2, 5, i + 1)\n",
    "      \n",
    "# #     # Rescale the weights to be between 0 and 255\n",
    "# #     wimg = 255.0 * (w[i, :, :, :].squeeze() - w_min) / (w_max - w_min)\n",
    "# #     plt.imshow(wimg.astype('uint8'))\n",
    "# #     plt.axis('off')\n",
    "# #     plt.title(classes[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}