Evaluate_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import load_model\n",
    "from sklearn.metrics import roc_curve, accuracy_score, classification_report, auc, roc_auc_score\n",
    "from LID_project.datasets.dataset import DataGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def val2onehot(val_array, classes):\n",
    "    labels = np.zeros((len(val_array), classes))\n",
    "    for ind,lbl in enumerate(val_array):\n",
    "        labels[ind,lbl] = 1\n",
    "    return labels\n",
    "\n",
    "def EER(true_targets_onehot, predictions):\n",
    "    '''\n",
    "    Imputs :\n",
    "        true_targets_onehot: one hot encoding of true values, shape (n_samples x n_classes)\n",
    "        predictions: one hot encoding of models prediction, shape (n_samples x n_classes)\n",
    "    Output:\n",
    "        per class EER score, np vector of length (n_clases,)\n",
    "\n",
    "    !!!!Be carefull!!! check that the target_to_class dictionary is the same for the training and the test, \n",
    "    if not you are returning incorect classes\n",
    "    Use predictions = model.predict(data_gen_val, verbose=1) to get the predictions,\n",
    "    and true_targets_onehot = val2onehot(data_gen_val.getTargets(), 8) to get the true targets\n",
    "\n",
    "    '''\n",
    "    scores = []\n",
    "    for i in range(true_targets_onehot.shape[1]):\n",
    "        fpr, tpr, threshold = roc_curve(true_targets_onehot[:,i], predictions[:,i])\n",
    "        fnr = 1 - tpr\n",
    "        #print(1-roc_auc_score(true_targets_onehot[:,i], predictions[:,i]))\n",
    "        ballanced_thres_pos = np.nanargmin(np.absolute((fnr - fpr)))\n",
    "        scores.append(fpr[ballanced_thres_pos])\n",
    "    return np.array(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_model('best_of_the_best_model_lstm.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_gen = DataGenerator('train', batch_size=1, shuffle=False, net='lstm', feat='plp') # has to be changed with \n",
    "data_gen_test = DataGenerator('test', batch_size=1, shuffle=False, net='lstm', feat='plp') # has to be changed with "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_gen_test.target_to_class = data_gen.target_to_class # making the dictionary the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16000/16000 [==============================] - 1193s 75ms/step\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict(data_gen_test, verbose=1) # get predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(open('pred.npy', 'wb'), predictions) # save predictions in case something goes wrong"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_targets_onehot = val2onehot(data_gen_test.getTargets(), 8) # finding true targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(true_targets_onehot, (predictions.T == np.max(predictions, axis=1).T).T*1, digits=3)) # get classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = EER(true_targets_onehot, predictions)\n",
    "print(l)\n",
    "print(np.mean(l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}