ALiPY Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>numqdata</th>\n",
       "      <th>round</th>\n",
       "      <th>init_labeled_size</th>\n",
       "      <th>num_of_queries</th>\n",
       "      <th>cost</th>\n",
       "      <th>performance</th>\n",
       "      <th>performance_plus_minus</th>\n",
       "      <th>dataset_id</th>\n",
       "      <th>strategy_id</th>\n",
       "      <th>dataset_random_seed</th>\n",
       "      <th>...</th>\n",
       "      <th>n_redundant</th>\n",
       "      <th>n_repeated</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>n_clusters_per_class</th>\n",
       "      <th>weights</th>\n",
       "      <th>flip_y</th>\n",
       "      <th>class_sep</th>\n",
       "      <th>hypercube</th>\n",
       "      <th>scale</th>\n",
       "      <th>random_state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.929415</td>\n",
       "      <td>0.014659</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.903619</td>\n",
       "      <td>0.085935</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.840795</td>\n",
       "      <td>0.169882</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.903605</td>\n",
       "      <td>0.041869</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.787143</td>\n",
       "      <td>0.217998</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19519</th>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0.763469</td>\n",
       "      <td>0.024513</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19520</th>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0.763469</td>\n",
       "      <td>0.024513</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19521</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.871350</td>\n",
       "      <td>0.067123</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19522</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.871350</td>\n",
       "      <td>0.067123</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19523</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.871350</td>\n",
       "      <td>0.067123</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16566 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       numqdata  round  init_labeled_size  num_of_queries  cost  performance  \\\n",
       "0            48      0                  2              10     0     0.929415   \n",
       "1            48      0                  2              10     0     0.903619   \n",
       "2            48      0                  2              10     0     0.840795   \n",
       "3            48      0                  2              10     0     0.903605   \n",
       "4            48      0                  2              10     0     0.787143   \n",
       "...         ...    ...                ...             ...   ...          ...   \n",
       "19519       250      0                  2              50     0     0.763469   \n",
       "19520       250      0                  2              50     0     0.763469   \n",
       "19521        48      0                  2              10     0     0.871350   \n",
       "19522        48      0                  2              10     0     0.871350   \n",
       "19523        48      0                  2              10     0     0.871350   \n",
       "\n",
       "       performance_plus_minus  dataset_id  strategy_id  dataset_random_seed  \\\n",
       "0                    0.014659           1            3                    2   \n",
       "1                    0.085935           1            2                    9   \n",
       "2                    0.169882           1            3                    4   \n",
       "3                    0.041869           1            2                    7   \n",
       "4                    0.217998           1            2                    6   \n",
       "...                       ...         ...          ...                  ...   \n",
       "19519                0.024513           2            3                  100   \n",
       "19520                0.024513           2            3                  100   \n",
       "19521                0.067123           1            3                  100   \n",
       "19522                0.067123           1            3                  100   \n",
       "19523                0.067123           1            3                  100   \n",
       "\n",
       "       ... n_redundant  n_repeated  n_classes  n_clusters_per_class  weights  \\\n",
       "0      ...           0           0          2                     0        0   \n",
       "1      ...           0           0          2                     0        0   \n",
       "2      ...           0           0          2                     0        0   \n",
       "3      ...           0           0          2                     0        0   \n",
       "4      ...           0           0          2                     0        0   \n",
       "...    ...         ...         ...        ...                   ...      ...   \n",
       "19519  ...           0           0          2                     0        0   \n",
       "19520  ...           0           0          2                     0        0   \n",
       "19521  ...           0           0          2                     0        0   \n",
       "19522  ...           0           0          2                     0        0   \n",
       "19523  ...           0           0          2                     0        0   \n",
       "\n",
       "       flip_y  class_sep  hypercube  scale  random_state  \n",
       "0         0.0        0.0          0    0.0             2  \n",
       "1         0.0        0.0          0    0.0             9  \n",
       "2         0.0        0.0          0    0.0             4  \n",
       "3         0.0        0.0          0    0.0             7  \n",
       "4         0.0        0.0          0    0.0             6  \n",
       "...       ...        ...        ...    ...           ...  \n",
       "19519     0.0        0.0          0    0.0           100  \n",
       "19520     0.0        0.0          0    0.0           100  \n",
       "19521     0.0        0.0          0    0.0           100  \n",
       "19522     0.0        0.0          0    0.0           100  \n",
       "19523     0.0        0.0          0    0.0           100  \n",
       "\n",
       "[16566 rows x 26 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import argparse\n",
    "import csv\n",
    "\n",
    "import math\n",
    "import multiprocessing\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "from timeit import default_timer as timer\n",
    "from operator import itemgetter\n",
    "#import dill\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import pairwise_distances\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.preprocessing import MinMaxScaler, RobustScaler\n",
    "\n",
    "from alipy.data_manipulate.al_split import split\n",
    "from alipy.experiment.al_experiment import AlExperiment\n",
    "from alipy.index import IndexCollection\n",
    "import timeit\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "font_size = 8\n",
    "\n",
    "tex_fonts = {\n",
    "    # Use LaTeX to write all text\n",
    "    \"text.usetex\": True,\n",
    "    \"font.family\": \"times\",\n",
    "    \"axes.labelsize\": font_size,\n",
    "    \"font.size\": font_size,\n",
    "    \"legend.fontsize\": font_size,\n",
    "    \"xtick.labelsize\": font_size,\n",
    "    \"ytick.labelsize\": font_size,\n",
    "    \"xtick.bottom\":True,\n",
    "    \"figure.autolayout\": True,\n",
    "    \n",
    "}\n",
    "\n",
    "sns.set_style('white')\n",
    "sns.set_context(\"paper\")\n",
    "plt.rcParams.update(tex_fonts)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# https://jwalton.info/Embed-Publication-Matplotlib-Latex/\n",
    "def set_size(width, fraction=1):\n",
    "    \"\"\"Set figure dimensions to avoid scaling in LaTeX.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    width: float\n",
    "            Document textwidth or columnwidth in pts\n",
    "    fraction: float, optional\n",
    "            Fraction of the width which you wish the figure to occupy\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    fig_dim: tuple\n",
    "            Dimensions of figure in inches\n",
    "    \"\"\"\n",
    "    # Width of figure (in pts)\n",
    "    fig_width_pt = width * fraction\n",
    "\n",
    "    # Convert from pt to inches\n",
    "    inches_per_pt = 1 / 72.27\n",
    "\n",
    "    # Golden ratio to set aesthetic figure height\n",
    "    # https://disq.us/p/2940ij3\n",
    "    golden_ratio = (5**.5 - 1) / 2\n",
    "\n",
    "    # Figure width in inches\n",
    "    fig_width_in = fig_width_pt * inches_per_pt\n",
    "    # Figure height in inches\n",
    "    fig_height_in = fig_width_in * golden_ratio\n",
    "\n",
    "    fig_dim = (fig_width_in, fig_height_in)\n",
    "\n",
    "    return fig_dim\n",
    "\n",
    "width = 505.89\n",
    "\n",
    "\n",
    "df = pd.read_csv(\"result.csv\")\n",
    "\n",
    "# rename to more meaningful names\n",
    "df['strategy'] = df['strategy'].replace({\n",
    "    \"QueryInstanceUncertainty{'measure': 'margin'}\":\"MM\", \n",
    "    \"QueryInstanceUncertainty{'measure': 'least_confident'}\":\"LC\", \n",
    "    \"QueryInstanceUncertainty{'measure': 'entropy'}\":\"Ent\", \n",
    "    \"QueryInstanceGraphDensity{}\":\"GD\", \n",
    "    \"QueryInstanceRandom{}\":\"Rand\", \n",
    "    \"QueryInstanceQUIRE{}\":\"QUIRE\", \n",
    "    \"QueryInstanceQBC{}\":\"QBC\", \n",
    "    \"QureyExpectedErrorReduction{}\":\"EER\", \n",
    "    \"QueryInstanceLAL{}\": \"LAL\",\n",
    "    \"QueryInstanceSPAL{}\": \"SPAL\",\n",
    "    \"QueryInstanceBMDR{}\": \"BMDR\",\n",
    "    \n",
    "      \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '../trained strategies/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Single',\n",
    "      \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '../trained strategies/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Single cos',\n",
    "      \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '../trained strategies/batch.pickle', 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Batch',\n",
    "})\n",
    "\n",
    "\n",
    "# for large datasets the cosine variants work better than the single ones\n",
    "large_datasets_where_cosine_is_better_for_single = [7,8,10,11]\n",
    "\n",
    "for dataset_id in large_datasets_where_cosine_is_better_for_single:\n",
    "    df = df[(df.strategy != 'NN Single') | (df.dataset_id != dataset_id)]\n",
    "\n",
    "df = df[(df.strategy != 'NN Single cos') | (df.dataset_id.isin(large_datasets_where_cosine_is_better_for_single))]\n",
    "\n",
    "\n",
    "df['strategy'] = df['strategy'].replace({\n",
    "      'NN Single cos': 'NN Single',\n",
    "})\n",
    "\n",
    "\n",
    "df = df[df.strategy != 'NN Batch']\n",
    "df['strategy'] = df['strategy'].replace({'NN Single': 'ImitAL'})\n",
    "\n",
    "df = df[df.dataset_id != 0]\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAACYCAYAAAAm509vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAU6ElEQVR4nO2dXWzk1nmGX0m72l17hyPbV06Xcgs0Vj2kCrQB0obqRQMYLccGEniLlIs2dVZppgOjhlQE0p1XQOUtYEBo0BXQH+00XcVoCvEiTVA0M6dIi8KAl+u49V81VOFb0UkvWici1/GmrQP1Qj1nSQ5nOKMhh0PO9wALi4c/c+Z4Ph7ynPe839Tx8fExCIIoLNNZV4AgiHShICeIgkNBThAFh4KcIAoOBTlBFJwzWVdgEBYWFrKuAkGMLe+++25kea6CHAAO3nyr676pmRlMnz83wtoQxHjQqwPMXZD/4OW/7brv4Wcvj7AmBJEPhnont20bTz75JCzLgmVZME1z4Gt4noeVlZVhqtEBr49t23AcB5cvX8bGxgYcx+nYJoiiM1RPrigKZFmGpmkAgOXlZVSrVUiS1Pc1JEnC3NzcMNUIwBiDLMtQFAWmaULTNOzu7oo6OY4T2CaIopPI4zrvMTVNgyRJcBxHlNVqNTDG0Gw2ceXKFdi2jVqtBsdxYFkWVFXF0dFREtUAAGiahqtXr0JVVaytrcF1XbTbbXieB1mWIUlSYFtRlMQ+myDGkUSm0BRFgSRJoneUZRm6rkOSJDDGoGka5ubmoGmaeETe2tpCtVqFoiiJ9uSO42BtbQ2lUgk7OzviSUPXdZim2bFNEEUnsXlyTdNE0DDGYJomyuWy2F8qlZL6qJ5YlgVN07C+vg4AgUB2HKdjmyCKzlCP63wgy3EcyLKMS5cugTGGDz/8EOVyGa7r4vDwUDzC+/+tr6+j1WpBlmW0221xjWHRNE28ly8tLUFVVdi2jXa7jbW1NciyHNgmiKIzlaelpgsLC3j1D6513f/ws5cx8+ADI6wRQYwHCwsLxRHD9JoLn5qZGWFNCCIf5C7IqacmiMHIXZB/9MEP+jpu6sxZzJwfzWAfQYwzsUFu2zZWV1dx48aNnnPKnueh3W4LYYwfx3HAGEOtVhPby8vL2NzcjDy+F9//q9/r67iLn/tjfOPv98RceLlcFnPziqJAVVVYlgXXdWEYxkB1IIg8ETuFxlVtcaIRSZLEPHh4/pkHF0eW5YBSLg0at16GruvQdR2NRkOM5NdqNTQaDUiSBEVR4HleanUgiHFgoHlyxhhWVlZgWRa2trbEfwGIv23bFmo3P7Isj3Rees43R99ut2EYBiRJgm3bqFQqI6sHQWTNQEEeVq5pmob5+XnYti16ZU3TRE8NnDzuq6oKXdext7cXed00etPPXf6sWKjiF+U0m00hlCGISWBgxRtXrsVJUXmv7TgO2u02AODOnTuRx/of5ZPEMAyoqip6bsYY6vU6Kd2IiaKvgTe+4MR1XaFY4yo127YBQOyTJAlHR0fwPE/0pJubmwCAcrksVobx93TP89BsNqHreqJf7L33vo/X3/l3uK6Ler0ulsI2m03Isoz19XVYloX9/f3E1HYEMY7kTvH2nd9f7OvYj33xJs5cfDjlGhHEeFAoxdvHvnizr+OmzpxNuSYEkQ9yF+TUOxPEYOQuyH/8o/cHPmd6ZhazpH4jJpTcBfkre78z8Dm/8PSf4c23b3eo2/jgIAChigtvcxcZLgoiiLyReHIFbpTon6ayLAvLy8vwPC92fxpIpVKkuo2P9HNVXHibj7oritJ1jp8gxp3Eg1yWZRiGAcaYKPM8D5VKBZIkxe4fJfv7+wGDx/A2780ZY7hy5cpI60YQSZFKmiS/hDVqDjpu/zjBNe7+mxJB5InUcqEpigLbtoUr6qD7R8Hi4qJ4hJdluWObL7TRNC01VR5BpE1qA2+GYWBjY6PrMs64/UnjV7dx5Z1hGGi1WiiXyzAMA7IsB7aBE8WfZVlimSxB5I3Eg9y2bTDGoKoqSv8/4MUYw8HBgRh467U/rfdywzACNxRuGR2+yUTddMibncgzuZO1vvPm4I/NUfPktm1T8BKFoVCy1vMPPpLIdSjAiUkhd0H+ow8HV7z5mZk5i/PnKA8aMTnkLsj/+hu/NdT5n/21Hbz9Vqe3G1e/8fKwGs62bZimiVKphLm5ObF8liDGndSm0DiO44j0xn74AJ1lWWCModFopF0VABCDfWH1W9jzLax+k2UZt27dwvb2duJr3wkiTVIP8ijTRs/zYJomdF0XgZRkZtMkiFK/AfdzrRFEXkg9yKNotVodA1/1ej2LqgyE53nk7krkjpEHebcgGbVuPY6w+g046cXHrZ4EEcfIg5w/7nJvOH/5KOvA1W+e54nxAH85V8MxxsQAneu6AedXgsgDqYth/NlSuGnj9va2MIiUJAme5/U1mLWwsIAvv/DTQ9Xn87/xN3jwgeBc+7gvkiGIOHqJYXKneHvzreF6/Kh5clK/EXmnUIq3cC+cBBTgRJHJXZD/8N5wijcAODtzFhdnaQCNmAxyF+RfZr899DX+UPsLvNqH6g04EcWoqirSQlEmVCJvpDa6btu2ULpZloWVlZWxmWMuSf2p3hhjwuNN0zTKhErkklSdYbjSTVVVXL9+PTIHWVSq43HBsiyx/p3ncyOIvJHqPLnrurAsCzs7OwGvNJ7+uNFodE11PC7wjKyj0tYTRNKkGuTlclmkNwZO3nmj0h/7Ux2PE+NYJ4IYlJEo3vggFVe18fTHfkbdk/eremu322CMCY83/36CyAOpja7zFMc8sLk8lKc85v9c1xWpjkdJlOdbVHmUBxyNrBN5IneKN+2lnxn6Ol/Rv46HLgRFNaR6I/JMoRRvX9G/PvQ1zs50pjWmACeKSu6CPNwDEwTRm9wF+fv3Pkj0erMzMyjNXkj0mgQxTqQa5LZto91ui6ko27ZRq9Vg2zZWV1eFGSIv74dn2Z8nWsc/1T6P22+9Eilx5QOHXNizuroKVVVRq9XEd2o0GjAMg8wkiLEltSk0z/Ows7MDwzCgaRo0TRM+bn41HPd429raSqsqPekmcW21WpBlGbVaTUyv7e7uYnNzUwQ4nyEgiHEmtZ681WphaWkpUNbNx02WZRwcHKRVlVPhz4VWqVQAAO12WyRoVBSFzCaIXJBakIezlXK7p7yNYjebTayvrwO4r4Db2NgQTrNhGyuCGDdSe1zXdT3g2yZJEnZ2diKPzTJ9cS8YY6jX6x2LaHhmVK5+o7TGxDiTWk8uyzJ0XRfLNV3XFY/vYTWcbduZZiSJSmusKApM00Sz2YQsy6jX62IgcW1tLfAe77puZnUniDhyp3j72Ze+lOg1X9afwyMXLgbK6F2byBuFUry9rD+X6PVmZ2Y6ysgUgigSuQvycK+bBnkbHCSIXuQuyN+/92HWVcgds9MzKJ07l3U1iIxIJMj9iRL4OmwgmFgBAG7fvi2mo8LJFRzH6Uv19oVvfzOJKk8UX5i9iOY3v9WRdtlxnA4VHx8MLZfLkGU5oPqjRI/5ZOgg5xlK+Q/H8zxsbGwIZVilUhE/DsZYYATbP6KeleJtEvipS5dw69YtAJ3pqHZ3d4Uk129caZom2u02qtUqarUalpeXKchzytBB3mq1Av/zJUkKmB7evXtX9Nrc5olPUfnJQ1bTvLLwxBMAotMu+1V8mqbh6tWrUFUVa2trIvj9qj8ifwwthokbiS6VSuJRr5cDDC3wSJeotMs8sHVdh2macBwHa2trKJVKAeGSX/VH5I+hg5wnHADuB3xU5k9JkjA/Py96/iyzmk4iUWmXwyo+3tP7A9qv+iPyydCP67yX3trawvz8PA4PD3Hjxg0AJz+cg4MDEcCHh4eo1+uQJAmGYYAxNlBWU+L0uK4rBD58HMUwjICKD7j/Xr60tATLsgKqP+rN80niirdGo5HaSOzCwgI+/kcbiV+36Hzt6WfwyIUHAmWk6isWI1W8HR0dodFopDYS+7Wnn0nlukVmdppUfZNM7rTr3e5WBDHJFEq7/v69/866CsSYMDs9jdK5TuddIkjugvx3v/1q1lUgxoSvPv0rAO777AH3Z2m4xRjHr+TjYh+ekho4mWkIKwKLQmKmEY7jiFTFUfv8CQN7HUsQg+D32TNNU8z7+39vjDGRl4+nvfKnpJZlGbdu3cL29nYhZ3kSC3KuZosacPMbRMQdSxCD4J8l2N/fF1oA/7y+pmm4du0aNjY2UK1WO1JSc/VllCKwCIwk4SEAsdiBIJLiX157ra+gjFLyhVNSRykCi0KqlszAie6ZN+je3l7PYwliEKSyFPDZW1xcFL+l8Pu4X8kXpQ+IUgQWhdQG3izLgq7rInMpANy5c6fnsQQxCAtPVHD2f09mW3hyjFarhXK5DMMwhLJP07SAkk9VVbRarcDyZr8isGgkFuR+c0bP89BsNiFJJ3daPlrJl5hqmtZxLAU5cRokScL29rbYDqeV1nVdDLL5iUpJXVRyJ4ax3v63rKtBjAn9zJNPSkrqQolhHrlANkZE/0xCgMeRuyB3732UdRWIgnNmZgoPznbq/fNK7oL8GvuPrKtAFJwX9Uc7vO4sy+rIfBvlkQcEM92GlXZZEDuFZts2GGNibTGHq9a4+YNpmmg0GrBtW5Tzvy3LCpwfLl9ZWaFpNGJs+Kfv/ENAISdJUmTmW6B3ptuw0i4revbkcSaN/lHLarUqpiH85X5l2/LyMqrVaiB1sed5uH79OhzHofcnYiz45C99CivPfUl43fWiV6bbsGdeVvTsyeNMGvuFPw1omiYEB67rwrIs7OzsiDslQYwD3/vee5Fed2HCHnlhWWw3z7xR0zPIk3qEVhQFkiQFFEX8MWZ+fj7RzyKIYXn9tTsdXndRxGW6jfLMy4KeQR5n0lgqlcTfcXZCmqYFGoUTXiJIEFnzyV/+FBhjsG1bZOL1Z771PA+NRgPVahW2bcM0TZHpVlVVACdPqlxp579OFvR8J+9l0ggA6+vrME0zENzcY922bZFRhd8ALl26BMYYHnvsscDKNMZYoRVHRL74uScqKP/izwfKDMMI/EZ1XRevmf5XzbACbxxeQwdSvKVp0tgPCwsL+PWX/jGTzyYmhxf1R1G+0Ht2edyUdIkp3tI2aeyHF/VHM/tsYjI4MzMVe8w4BXgcAwV51gMIAGLvsARBBMldxPz43k+yrgIx4UxPT2H23Mj8VoYmsdTFq6urQjSzt7cnBC79lPvTHcfxz3/3wySqTBCn5hO/OoM33/qukLl6nhdpIOmXt3J6GUiura1FymeHJZHbkV/BpqpqQMEWV86nGWienMgLYZlrlIGkX97KiTOQ7CWfHYbEnjm6Kdh6KduilHAEkTeiDCSjdCNZGUgmFuTdFGy9lG1RSjiCyDu9gjYLA8nERw+6Kdi6lXdTwhFEXggbSIblrZysDCQTGXjz+7sB9xVsceVhJRw9thN5wS9zDRtI+t+rXdcNpIqOM5D0XzcpY8ncebz9yebtrKtBTDif/sxDOH9hMOeYtFNFF8rj7dOfeSjrKhATzvR0vCIuTJazR7kL8kHvoAQxDmQpg81dkH/0ASneiPFiamYKMxfGVwGXuyB/7y//M+sqEEQAyTiH118JKuD4gDNfYx6lZONKOf953E5KURSUy+XAdU47l97X7cefatiyLGxtbYnyy5cvB5Q9lmVheXkZnud1GDbyucFuBo8EkUekUlCp1mq1IMsyarUaGo1GVyVbuJwPzimKgr29vY7rnJa+glyWZVQqFSFDvXv3rqiQYRhgjIljPc9DpVIRX8AvX9V1HVtbWyRrJQoN16vbto1KpdL3ebz3ZozhypUrp75OmL5fJO7evStkqNyRFQimJI6bJpBlGQcHB2KbZK1EkWk2mwMvz+ado7/jPM11/PQd5KVSSbwXHB0ddchTbdsW1rT9QrJWoqgwxlCv1zsWqfSCv7b6vRVPc50wAw8JSpKE+fl5tFotUWYYRl/v1VE3AZK1EkXAr1Tj40wvvPAC9vb2OvZzI8hwuaqqsG0bjUYDtVot8jqnoa/RdcdxcHBwIO4uh4eHqNfr4nFbVVXR0zPGcHBwAM/z4LpuQNZq2zY2NzdJ1koUDr/Roz+hSNR+ACJVd7gcCM6pJ7E6LXeyVvuNg/gDCWKEnGaePGkjyELJWs9cJMUbkX9GqYDLXZD/5IP/yboKBNGVqTNTmD5/NutqBMhdkP/XV9/IugoE0ZXZ3/w4vvvOv/ZUv/GEJH4/OMZYh99b+LzTqt9OLbjlkjy/Aq6bwg04GbwLb/NjCaIoxKnfdnZ2oOt6wB0GQIffW/i8YdRvp+7JW60WqtUqJEkSX8ivZAMg8kDxlK6WZYnF8lxQk2WiBoJIGz5yzlVrc3NzYp8/Q3A3vzd+Xvg6g3DqIFdVFVevXkW1Wo20j/U8D0dHR4EBBq6OS3PxPEGMI1y15nme6JX9yUOBaL+3sNrtNOq3Uz+uK4qC3d1dyLKM1dVVUe66Lmzbhuu6IvGbbdvCwK7bpD5p14miElatGYYBVVU7euSw31v4vNOq307dk/PHCj6p73keJElCuVzumB5wHAeu6wIA7ty50/V6/FoEkWf8KjbHcWCaJprNJmRZxlNPPSXioV6vC/+3Wq0W8Hvjajd+3tLSUmB7kN781EHO/aMlSYLruoE0xf6Jfj4Ix7OllMtlYUbP39M9z0Oz2aQgJwpBnPot3An61W8cvkLTz2nHr3KneDt4Yz/rahBEV04zT56E+q1QirfKJxazrgJB5IvjHPLMM8/0vf3444+PrB5JHd/ruG774tokXBben1Y7DdpG/Z6TRBtFlWXxW0qrjTjj6z5HEEQi5DLIw/PycdujqkdSx/c6rtu+ftrAXzaubdTvOUm0UVRZFr+ltNqIk6uBt9PQa0CCuA+1Uzx5baNc9uSD8Pzzz2ddhVxA7RRPXtuo8D05QUw6he/JCWLSoSAniIKTOzGMn3CamXA5cN+wvtd2EVfFRaXgifrOk9hWcW3T7Tvntq0GnoUfMw4PD49v3rwZKLt58+ax67rHx8fHx9euXYvdLir+tun2nSe1rXq1TdHaqpCP6/v7+2LJnuM4sduTQLfvTG0V3wb9HjeubVXIICcI4j6FDPLFxUVhQiHLcuz2JNDtO1NbxbdBv8eNa1vlfp7cNE3cvn0b6+vrYq26YRhotVool8vCS67X9ig9sEdJuG3CbTDJbRXXNkVqq9wHeRjykOsfaqv+yXNbFe5xnbzi+ofaqn/y3FaF68kJgghSuJ6cIIggFOQEUXAoyAmi4FCQE0TBoSAniILzf/rb2arBTAZCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x155.745 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def show_values_on_bars(axs, h_v=\"v\", space=0.4):\n",
    "    def _show_on_single_plot(ax):\n",
    "        if h_v == \"v\":\n",
    "            for p in ax.patches:\n",
    "                _x = p.get_x() + p.get_width() / 2\n",
    "                _y = p.get_y() + p.get_height()\n",
    "                value = int(p.get_height())\n",
    "                ax.text(_x, _y, \"{:,.0f}\".format(value), ha=\"center\") \n",
    "        elif h_v == \"h\":\n",
    "            for p in ax.patches:\n",
    "                _x = p.get_x() + p.get_width() + float(space)\n",
    "                _y = p.get_y() + p.get_height() - 0.15\n",
    "                value = int(p.get_width())\n",
    "                ax.text(_x, _y, \"{:,.0f}\".format(value), ha=\"left\")\n",
    "\n",
    "    if isinstance(axs, np.ndarray):\n",
    "        for idx, ax in np.ndenumerate(axs):\n",
    "            _show_on_single_plot(ax)\n",
    "    else:\n",
    "        _show_on_single_plot(axs)\n",
    "\n",
    "\n",
    "plt.figure(figsize=set_size(width, fraction=0.5))\n",
    "from numpy import median\n",
    "from matplotlib.ticker import ScalarFormatter,AutoMinorLocator\n",
    "\n",
    "\n",
    "df_dur = df.groupby(['strategy', 'dataset_id'])['duration'].mean().to_frame()\n",
    "df_dur = df_dur.reset_index()  \n",
    "\n",
    "\n",
    "df_dur2 =  df.groupby(['strategy'])['duration'].mean().to_frame()\n",
    "\n",
    "timout_duration = 604800\n",
    "#timout_duration = 259200\n",
    "\n",
    "# timeout\n",
    "df_dur = df_dur.append({'strategy':'LAL', 'dataset_id':'8', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'EER', 'dataset_id':'8', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'QUIRE', 'dataset_id':'8', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'BMDR', 'dataset_id':'8', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'SPAL', 'dataset_id':'8', 'duration':timout_duration}, ignore_index=True)\n",
    "\n",
    "df_dur = df_dur.append({'strategy':'BMDR', 'dataset_id':'7', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'SPAL', 'dataset_id':'7', 'duration':timout_duration}, ignore_index=True)\n",
    "\n",
    "\n",
    "df_dur = df_dur.append({'strategy':'EER', 'dataset_id':'11', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'QUIRE', 'dataset_id':'11', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'BMDR', 'dataset_id':'11', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'SPAL', 'dataset_id':'11', 'duration':timout_duration}, ignore_index=True)\n",
    "\n",
    "\n",
    "df_hist_new = pd.DataFrame(columns=['strategy', 'duration'])\n",
    "for strategy in df.strategy.unique():\n",
    "    df_hist_new = df_hist_new.append({\n",
    "        'strategy':strategy, \n",
    "        'duration': df_dur.loc[df_dur['strategy'] ==strategy]['duration'].mean()\n",
    "    }, ignore_index=True)\n",
    "\n",
    "\n",
    "df_hist_new = df_hist_new.sort_values(by=['duration'])\n",
    "g = sns.barplot(data=df_hist_new, x=\"duration\", y=\"strategy\")#, ci=68, capsize=.3)#, fill=True, common_norm=False, alpha=.5, linewidth=0)\n",
    "show_values_on_bars(g, \"h\", 10.4)\n",
    "g.set_xscale(\"log\")\n",
    "#g.xaxis.set_minor_locator(AutoMinorLocator(5))\n",
    "\n",
    "#print(g.get_xlim())\n",
    "g.set_xlim(g.get_xlim()[0],g.get_xlim()[1]+170000)\n",
    "\n",
    "import matplotlib.ticker\n",
    "g.get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())\n",
    "g.get_xaxis().set_minor_formatter(matplotlib.ticker.NullFormatter())\n",
    "g.get_xaxis().set_major_formatter(\n",
    "    matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))\n",
    "\n",
    "g.xaxis.tick_bottom()\n",
    "g.set_xlabel(\"\")\n",
    "#g.set_xlabel(\"duration in s (walltime)\")\n",
    "g.set_ylabel(\"\")\n",
    "g.get_figure().savefig('performance.pdf', format='pdf', bbox_inches='tight')\n",
    "g\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1  3  2 12 15  4  5  6  7  8  9 10 11 13 14]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>strategy  </th><th>ImitAL   </th><th>MM        </th><th>QBC      </th><th>LC       </th><th>GD       </th><th>BMDR      </th><th>Ent       </th><th>LAL       </th><th>SPAL      </th><th>EER       </th><th>QUIRE     </th><th>Rand      </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>australian</td><td>85.4% (0)</td><td>85.4% (2) </td><td>85.3% (3)</td><td>85.4% (1)</td><td>85.2% (4)</td><td>85.1% (6) </td><td>85.4% (2) </td><td>85.2% (5) </td><td>84.3% (9) </td><td>84.6% (8) </td><td>76.2% (10)</td><td>84.9% (7) </td></tr>\n",
       "<tr><td>CIFAR-10  </td><td>37.0% (0)</td><td>36.9% (1) </td><td>36.0% (4)</td><td>35.9% (5)</td><td>36.3% (3)</td><td>0.0% (8)  </td><td>35.7% (6) </td><td>34.9% (7) </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>36.8% (2) </td></tr>\n",
       "<tr><td>diabetes  </td><td>74.3% (3)</td><td>74.3% (4) </td><td>74.4% (2)</td><td>74.3% (5)</td><td>75.0% (0)</td><td>74.7% (1) </td><td>74.3% (4) </td><td>74.3% (6) </td><td>73.9% (9) </td><td>74.2% (8) </td><td>66.5% (10)</td><td>74.2% (7) </td></tr>\n",
       "<tr><td>DWTC      </td><td>74.9% (0)</td><td>74.7% (1) </td><td>71.7% (5)</td><td>71.0% (6)</td><td>65.1% (8)</td><td>0.0% (10) </td><td>68.5% (7) </td><td>72.2% (4) </td><td>0.0% (10) </td><td>72.8% (3) </td><td>53.4% (9) </td><td>72.9% (2) </td></tr>\n",
       "<tr><td>EMNIST    </td><td>69.1% (1)</td><td>69.5% (0) </td><td>63.4% (3)</td><td>61.6% (4)</td><td>43.8% (6)</td><td>0.0% (7)  </td><td>59.2% (5) </td><td>0.0% (7)  </td><td>0.0% (7)  </td><td>0.0% (7)  </td><td>0.0% (7)  </td><td>66.1% (2) </td></tr>\n",
       "<tr><td>fertility </td><td>88.5% (3)</td><td>87.9% (9) </td><td>88.2% (5)</td><td>88.0% (7)</td><td>88.3% (4)</td><td>88.6% (2) </td><td>88.0% (8) </td><td>88.0% (6) </td><td>88.6% (1) </td><td>86.6% (11)</td><td>89.1% (0) </td><td>87.4% (10)</td></tr>\n",
       "<tr><td>flags     </td><td>57.7% (0)</td><td>56.6% (4) </td><td>55.9% (8)</td><td>55.4% (9)</td><td>57.0% (2)</td><td>57.4% (1) </td><td>54.7% (10)</td><td>55.9% (7) </td><td>56.9% (3) </td><td>56.0% (6) </td><td>50.4% (11)</td><td>56.2% (5) </td></tr>\n",
       "<tr><td>german    </td><td>75.7% (2)</td><td>75.3% (10)</td><td>75.5% (5)</td><td>75.4% (8)</td><td>76.2% (1)</td><td>75.6% (4) </td><td>75.4% (9) </td><td>75.5% (6) </td><td>76.5% (0) </td><td>75.4% (7) </td><td>70.9% (11)</td><td>75.6% (3) </td></tr>\n",
       "<tr><td>glass     </td><td>69.0% (0)</td><td>68.6% (1) </td><td>67.5% (5)</td><td>68.0% (2)</td><td>66.4% (7)</td><td>67.6% (4) </td><td>66.3% (8) </td><td>66.0% (9) </td><td>61.7% (10)</td><td>67.2% (6) </td><td>48.2% (11)</td><td>67.6% (3) </td></tr>\n",
       "<tr><td>haberman  </td><td>72.9% (4)</td><td>72.9% (3) </td><td>73.3% (0)</td><td>72.9% (1)</td><td>71.6% (7)</td><td>71.6% (8) </td><td>72.9% (2) </td><td>72.3% (5) </td><td>0.0% (11) </td><td>72.0% (6) </td><td>63.2% (10)</td><td>71.5% (9) </td></tr>\n",
       "<tr><td>heart     </td><td>79.4% (0)</td><td>79.1% (6) </td><td>79.1% (8)</td><td>79.1% (5)</td><td>79.2% (3)</td><td>79.3% (1) </td><td>79.1% (6) </td><td>79.3% (2) </td><td>78.6% (9) </td><td>79.1% (4) </td><td>74.7% (10)</td><td>79.1% (7) </td></tr>\n",
       "<tr><td>ionosphere</td><td>90.3% (0)</td><td>90.0% (2) </td><td>89.9% (4)</td><td>90.0% (1)</td><td>89.6% (6)</td><td>89.8% (5) </td><td>90.0% (1) </td><td>89.9% (3) </td><td>83.2% (9) </td><td>89.4% (8) </td><td>53.7% (10)</td><td>89.6% (7) </td></tr>\n",
       "<tr><td>olivetti  </td><td>72.2% (1)</td><td>72.3% (0) </td><td>72.1% (3)</td><td>72.1% (2)</td><td>65.5% (9)</td><td>64.3% (10)</td><td>70.7% (4) </td><td>65.6% (8) </td><td>66.5% (5) </td><td>65.7% (7) </td><td>51.2% (11)</td><td>66.1% (6) </td></tr>\n",
       "<tr><td>planning  </td><td>72.3% (2)</td><td>72.1% (3) </td><td>72.7% (1)</td><td>72.1% (3)</td><td>69.6% (7)</td><td>68.9% (8) </td><td>72.1% (4) </td><td>71.5% (6) </td><td>73.6% (0) </td><td>68.2% (10)</td><td>71.8% (5) </td><td>68.4% (9) </td></tr>\n",
       "<tr><td>zoo       </td><td>93.4% (0)</td><td>93.0% (3) </td><td>92.5% (9)</td><td>92.9% (6)</td><td>92.7% (8)</td><td>93.3% (1) </td><td>92.8% (7) </td><td>92.3% (10)</td><td>92.2% (11)</td><td>92.9% (5) </td><td>93.2% (2) </td><td>93.0% (4) </td></tr>\n",
       "<tr><td>mean %    </td><td>74.2% (0)</td><td>73.9% (1) </td><td>73.2% (2)</td><td>72.9% (3)</td><td>70.8% (6)</td><td>61.1% (9) </td><td>72.3% (5) </td><td>68.2% (7) </td><td>55.7% (11)</td><td>65.6% (8) </td><td>57.5% (10)</td><td>72.6% (4) </td></tr>\n",
       "<tr><td>mean (r)  </td><td>1.07     </td><td>3.27      </td><td>4.33     </td><td>4.33     </td><td>5.00     </td><td>5.07      </td><td>5.53      </td><td>6.07      </td><td>6.80      </td><td>6.93      </td><td>8.33      </td><td>5.53      </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "'<table>\\n<thead>\\n<tr><th>strategy  </th><th>ImitAL   </th><th>MM        </th><th>QBC      </th><th>LC       </th><th>GD       </th><th>BMDR      </th><th>Ent       </th><th>LAL       </th><th>SPAL      </th><th>EER       </th><th>QUIRE     </th><th>Rand      </th></tr>\\n</thead>\\n<tbody>\\n<tr><td>australian</td><td>85.4% (0)</td><td>85.4% (2) </td><td>85.3% (3)</td><td>85.4% (1)</td><td>85.2% (4)</td><td>85.1% (6) </td><td>85.4% (2) </td><td>85.2% (5) </td><td>84.3% (9) </td><td>84.6% (8) </td><td>76.2% (10)</td><td>84.9% (7) </td></tr>\\n<tr><td>CIFAR-10  </td><td>37.0% (0)</td><td>36.9% (1) </td><td>36.0% (4)</td><td>35.9% (5)</td><td>36.3% (3)</td><td>0.0% (8)  </td><td>35.7% (6) </td><td>34.9% (7) </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>36.8% (2) </td></tr>\\n<tr><td>diabetes  </td><td>74.3% (3)</td><td>74.3% (4) </td><td>74.4% (2)</td><td>74.3% (5)</td><td>75.0% (0)</td><td>74.7% (1) </td><td>74.3% (4) </td><td>74.3% (6) </td><td>73.9% (9) </td><td>74.2% (8) </td><td>66.5% (10)</td><td>74.2% (7) </td></tr>\\n<tr><td>DWTC      </td><td>74.9% (0)</td><td>74.7% (1) </td><td>71.7% (5)</td><td>71.0% (6)</td><td>65.1% (8)</td><td>0.0% (10) </td><td>68.5% (7) </td><td>72.2% (4) </td><td>0.0% (10) </td><td>72.8% (3) </td><td>53.4% (9) </td><td>72.9% (2) </td></tr>\\n<tr><td>EMNIST    </td><td>69.1% (1)</td><td>69.5% (0) </td><td>63.4% (3)</td><td>61.6% (4)</td><td>43.8% (6)</td><td>0.0% (7)  </td><td>59.2% (5) </td><td>0.0% (7)  </td><td>0.0% (7)  </td><td>0.0% (7)  </td><td>0.0% (7)  </td><td>66.1% (2) </td></tr>\\n<tr><td>fertility </td><td>88.5% (3)</td><td>87.9% (9) </td><td>88.2% (5)</td><td>88.0% (7)</td><td>88.3% (4)</td><td>88.6% (2) </td><td>88.0% (8) </td><td>88.0% (6) </td><td>88.6% (1) </td><td>86.6% (11)</td><td>89.1% (0) </td><td>87.4% (10)</td></tr>\\n<tr><td>flags     </td><td>57.7% (0)</td><td>56.6% (4) </td><td>55.9% (8)</td><td>55.4% (9)</td><td>57.0% (2)</td><td>57.4% (1) </td><td>54.7% (10)</td><td>55.9% (7) </td><td>56.9% (3) </td><td>56.0% (6) </td><td>50.4% (11)</td><td>56.2% (5) </td></tr>\\n<tr><td>german    </td><td>75.7% (2)</td><td>75.3% (10)</td><td>75.5% (5)</td><td>75.4% (8)</td><td>76.2% (1)</td><td>75.6% (4) </td><td>75.4% (9) </td><td>75.5% (6) </td><td>76.5% (0) </td><td>75.4% (7) </td><td>70.9% (11)</td><td>75.6% (3) </td></tr>\\n<tr><td>glass     </td><td>69.0% (0)</td><td>68.6% (1) </td><td>67.5% (5)</td><td>68.0% (2)</td><td>66.4% (7)</td><td>67.6% (4) </td><td>66.3% (8) </td><td>66.0% (9) </td><td>61.7% (10)</td><td>67.2% (6) </td><td>48.2% (11)</td><td>67.6% (3) </td></tr>\\n<tr><td>haberman  </td><td>72.9% (4)</td><td>72.9% (3) </td><td>73.3% (0)</td><td>72.9% (1)</td><td>71.6% (7)</td><td>71.6% (8) </td><td>72.9% (2) </td><td>72.3% (5) </td><td>0.0% (11) </td><td>72.0% (6) </td><td>63.2% (10)</td><td>71.5% (9) </td></tr>\\n<tr><td>heart     </td><td>79.4% (0)</td><td>79.1% (6) </td><td>79.1% (8)</td><td>79.1% (5)</td><td>79.2% (3)</td><td>79.3% (1) </td><td>79.1% (6) </td><td>79.3% (2) </td><td>78.6% (9) </td><td>79.1% (4) </td><td>74.7% (10)</td><td>79.1% (7) </td></tr>\\n<tr><td>ionosphere</td><td>90.3% (0)</td><td>90.0% (2) </td><td>89.9% (4)</td><td>90.0% (1)</td><td>89.6% (6)</td><td>89.8% (5) </td><td>90.0% (1) </td><td>89.9% (3) </td><td>83.2% (9) </td><td>89.4% (8) </td><td>53.7% (10)</td><td>89.6% (7) </td></tr>\\n<tr><td>olivetti  </td><td>72.2% (1)</td><td>72.3% (0) </td><td>72.1% (3)</td><td>72.1% (2)</td><td>65.5% (9)</td><td>64.3% (10)</td><td>70.7% (4) </td><td>65.6% (8) </td><td>66.5% (5) </td><td>65.7% (7) </td><td>51.2% (11)</td><td>66.1% (6) </td></tr>\\n<tr><td>planning  </td><td>72.3% (2)</td><td>72.1% (3) </td><td>72.7% (1)</td><td>72.1% (3)</td><td>69.6% (7)</td><td>68.9% (8) </td><td>72.1% (4) </td><td>71.5% (6) </td><td>73.6% (0) </td><td>68.2% (10)</td><td>71.8% (5) </td><td>68.4% (9) </td></tr>\\n<tr><td>zoo       </td><td>93.4% (0)</td><td>93.0% (3) </td><td>92.5% (9)</td><td>92.9% (6)</td><td>92.7% (8)</td><td>93.3% (1) </td><td>92.8% (7) </td><td>92.3% (10)</td><td>92.2% (11)</td><td>92.9% (5) </td><td>93.2% (2) </td><td>93.0% (4) </td></tr>\\n<tr><td>mean %    </td><td>74.2% (0)</td><td>73.9% (1) </td><td>73.2% (2)</td><td>72.9% (3)</td><td>70.8% (6)</td><td>61.1% (9) </td><td>72.3% (5) </td><td>68.2% (7) </td><td>55.7% (11)</td><td>65.6% (8) </td><td>57.5% (10)</td><td>72.6% (4) </td></tr>\\n<tr><td>mean (r)  </td><td>1.07     </td><td>3.27      </td><td>4.33     </td><td>4.33     </td><td>5.00     </td><td>5.07      </td><td>5.53      </td><td>6.07      </td><td>6.80      </td><td>6.93      </td><td>8.33      </td><td>5.53      </td></tr>\\n</tbody>\\n</table>'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "dataset_to_id= {\n",
    "        1: \"fertility\",\n",
    "    2: \"german\",\n",
    "    3: \"diabetes\",\n",
    "    4: \"ionosphere\",\n",
    "    5: \"planning\",\n",
    "    6: \"australian\",\n",
    "    7: \"DWTC\",\n",
    "    8: \"EMNIST\",\n",
    "    9: \"glass\",\n",
    "    10: \"olivetti\",\n",
    "    11: \"CIFAR-10\",\n",
    "    12: \"haberman\",\n",
    "    13: \"zoo\",\n",
    "    14: \"flags\",\n",
    "    15: \"heart\",\n",
    "    \n",
    "}\n",
    "print(df['dataset_id'].unique())\n",
    "\n",
    "\n",
    "dataset_names = ['strategy'] + [dataset_to_id[id] for id in df['dataset_id'].unique()]\n",
    "strategy_results = []\n",
    "strategy_results.append(dataset_names)\n",
    "\n",
    "def get_ranks(l):\n",
    "    uniques = np.unique(l)\n",
    "    argssorted = np.flip(np.argsort(uniques))\n",
    "    mapping = dict(zip(uniques, argssorted))\n",
    "    ranks = [mapping[x] for x in l]\n",
    "    \n",
    "    m2 = {}\n",
    "    last_value = -1\n",
    "    increase = 0\n",
    "    for i, v in enumerate(sorted(ranks)):\n",
    "        if last_value == v:\n",
    "            increase += 1\n",
    "            m2[v] = v\n",
    "        else:\n",
    "            m2[v] = v+increase\n",
    "        last_value = v\n",
    "    \n",
    "    # uncomment to calculate ranks like : 0,1,1,3,4,5,5,7, so add +1 after a tie\n",
    "    # if it is uncommented we do not add an extra +1 after a tie (as it is done in the paper)\n",
    "    # as you can see, this has no influence of the superior ranking performance of NN Single, it only punishes all other techniques more\n",
    "    #ranks = [m2[x] for x in ranks]\n",
    "    return ranks\n",
    "\n",
    "for strategy in df['strategy'].unique():\n",
    "    results = [strategy]\n",
    "    for dataset in df['dataset_id'].unique():\n",
    "        results.append(\"{:.40f}\".format(df.loc[(df['dataset_id']==dataset) & (df['strategy']==strategy)]['f1_auc'].mean()))\n",
    "         \n",
    "        results = [r if r != \"nan\" else -0 for r in results] \n",
    "     \n",
    "    strategy_results.append(results)\n",
    "    \n",
    "results_df = pd.DataFrame(strategy_results[1:], columns=strategy_results[0])\n",
    "results_df = results_df.set_index(\"strategy\")\n",
    "results_df = results_df.replace(-1, np.NaN)\n",
    "results_df = results_df.apply(pd.to_numeric)\n",
    "\n",
    "results_df['mean'] = results_df.T.mean(skipna=True)\n",
    "\n",
    "\n",
    "ranking_only_table = []\n",
    "transposed_results = list(map(list, zip(*strategy_results)))\n",
    "new_results = [transposed_results[0]]\n",
    "for row in transposed_results[1:]:\n",
    "    row_sorting = get_ranks(np.array(row[1:]))\n",
    "    row = row[:1] + [\"{:.1%} ({})\".format(float(a),b) for a,b in zip(row[1:], row_sorting)]\n",
    "    new_results.append(row)\n",
    "    ranking_only_table.append(row[:1] + row_sorting)\n",
    "\n",
    "row = [\"mean %\"] + results_df['mean'].to_list()\n",
    "row_sorting = get_ranks(np.array(row[1:]))\n",
    "\n",
    "row = row[:1] + [\"{:.1%} ({})\".format(float(a),b) for a,b in zip(row[1:], row_sorting)]\n",
    "new_results.append(row)\n",
    "\n",
    "\n",
    "ranking_only_table = [transposed_results[0]] + ranking_only_table\n",
    "new_results.append([\"mean (r)\"] + [\"{:1.2f}\".format(sum(r)/len(r)) for r in list(map(list, zip(*ranking_only_table[1:])))[1:]])\n",
    "\n",
    "\n",
    "# sort colmuns manually\n",
    "df2 = pd.DataFrame(new_results[1:], columns=new_results[0])\n",
    "df2 = df2[['strategy', 'ImitAL', 'MM', 'QBC', 'LC', 'GD', 'BMDR', 'Ent', 'LAL', 'SPAL', 'EER', 'QUIRE','Rand']]\n",
    "\n",
    "order = ['strategy'] + sorted([v for v in dataset_to_id.values()], key=lambda v: v.upper()) + ['mean %', 'mean (r)']\n",
    "lol = [df2.columns.values.tolist()] + df2.values.tolist()\n",
    "lol = sorted(lol, key=lambda l: order.index(l[0]))\n",
    "\n",
    "with open('ali_f1auc_table.tex', \"w\") as f:\n",
    "    tex_code = tabulate(lol, headers=\"firstrow\", tablefmt=\"latex_booktabs\")\n",
    "    splitted = [\"\\\\fontseries{b}\\\\selectfont{\" + t[1:-1] + \"}\" if \"(0)\" in t else t for t in tex_code.split(\"&\") ]\n",
    "    \n",
    "    \n",
    "    tex_code = \"&\".join(splitted)\n",
    "    tex_code = tex_code.replace(\"llllllllllllll\", \"L{1.1cm}cccccccccccccc\")\n",
    "    \n",
    "    tex_code = tex_code.replace(\"tabular\", \"tabularx\")\n",
    "    tex_code = tex_code.replace(\"CIFAR-10\", \"\\mbox{CIFAR-10}\")\n",
    "\n",
    "    tex_code = tex_code.replace(\"begin{tabularx}\", \"begin{tabularx}{\\linewidth}\")\n",
    "\n",
    "    tex_code = tex_code.replace(\"mean \\%\", \"\\\\midrule \\n mean \\%\")\n",
    "    tex_code = tex_code.replace(\" \\\\% \", \"~~~\")\n",
    "    tex_code = tex_code.replace(\"\\\\% \", \" \")\n",
    "    tex_code = tex_code.replace(\"~~~\", \" \\\\%\")\n",
    "    for ind in range(0,100):\n",
    "        tex_code = tex_code.replace(\" 0.0 (\"+str(ind)+\")\", \"\")\n",
    "        \n",
    "    tex_code = tex_code.replace(\"1.36\", \"\\\\fontseries{b}\\\\selectfont{1.36}\")\n",
    "    tex_code = tex_code.replace(\" \\\\\\\\\", \"\\\\\\\\\")\n",
    "    tex_code = tex_code.replace(\"cc\", \"rr\")\n",
    "    tex_code = tex_code.replace(\"strategy\", \"\")\n",
    "    f.write(tex_code)\n",
    "    \n",
    "tabulate(lol, headers=\"firstrow\", tablefmt=\"html\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all together\n",
      "t = 14.31369908879011\n",
      "p = 7.016342322073623e-45\n",
      "Significant!\n",
      "fertility\n",
      "p = 0.15764253153954605\n",
      "Not significant\n",
      "\n",
      "diabetes\n",
      "p = 2.5169002061962217e-37\n",
      "Significant!\n",
      "\n",
      "german\n",
      "p = 3.919011768629743e-15\n",
      "Significant!\n",
      "\n",
      "haberman\n",
      "p = 1.5570115625436015e-15\n",
      "Significant!\n",
      "\n",
      "heart\n",
      "p = 3.1853195739726803e-28\n",
      "Significant!\n",
      "\n",
      "ionosphere\n",
      "p = 2.0895858911006804e-174\n",
      "Significant!\n",
      "\n",
      "planning\n",
      "p = 0.582420257544612\n",
      "Not significant\n",
      "\n",
      "australian\n",
      "p = 2.4209244924565232e-64\n",
      "Significant!\n",
      "\n",
      "DWTC\n",
      "p = 3.0803137839845545e-100\n",
      "Significant!\n",
      "\n",
      "EMNIST\n",
      "p = nan\n",
      "Not significant\n",
      "\n",
      "glass\n",
      "p = 4.2105224697392785e-70\n",
      "Significant!\n",
      "\n",
      "olivetti\n",
      "p = 3.857679791379383e-110\n",
      "Significant!\n",
      "\n",
      "CIFAR-10\n",
      "p = nan\n",
      "Not significant\n",
      "\n",
      "zoo\n",
      "p = 0.6849762356323811\n",
      "Not significant\n",
      "\n",
      "flags\n",
      "p = 1.0964499693905069e-31\n",
      "Significant!\n",
      "\n",
      "Means\n",
      "t = nan\n",
      "p = nan\n",
      "Not significant\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/julius/.local/share/virtualenvs/ALiPy-yBTkQe9L/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3372: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/home/julius/.local/share/virtualenvs/ALiPy-yBTkQe9L/lib/python3.8/site-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/home/julius/.local/share/virtualenvs/ALiPy-yBTkQe9L/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3372: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/home/julius/.local/share/virtualenvs/ALiPy-yBTkQe9L/lib/python3.8/site-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "dataset_to_id= {\n",
    "        1: \"fertility\",\n",
    "    2: \"german\",\n",
    "    3: \"diabetes\",\n",
    "    4: \"ionosphere\",\n",
    "    5: \"planning\",\n",
    "    6: \"australian\",\n",
    "    7: \"DWTC\",\n",
    "    8: \"EMNIST\",\n",
    "    9: \"glass\",\n",
    "    10: \"olivetti\",\n",
    "    11: \"CIFAR-10\",\n",
    "    12: \"haberman\",\n",
    "    13: \"zoo\",\n",
    "    14: \"flags\",\n",
    "    15: \"heart\",\n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "dataset_names = ['strategy'] + [dataset_to_id[id] for id in df['dataset_id'].unique()]\n",
    "strategy_results = []\n",
    "strategy_results.append(dataset_names)\n",
    "\n",
    "def get_ranks(l):\n",
    "    uniques = np.unique(l)\n",
    "    argssorted = np.flip(np.argsort(uniques))\n",
    "    mapping = dict(zip(uniques, argssorted))\n",
    "    ranks = [mapping[x] for x in l]\n",
    "    \n",
    "    m2 = {}\n",
    "    last_value = -1\n",
    "    increase = 0\n",
    "    for i, v in enumerate(sorted(ranks)):\n",
    "        if last_value == v:\n",
    "            increase += 1\n",
    "            m2[v] = v\n",
    "        else:\n",
    "            m2[v] = v+increase\n",
    "        last_value = v\n",
    "    \n",
    "    # uncomment to calculate ranks like : 0,1,1,3,4,5,5,7, so add +1 after a tie\n",
    "    # if it is uncommented we do not add an extra +1 after a tie (as it is done in the paper)\n",
    "    # as you can see, this has no influence of the superior ranking performance of NN Single, it only punishes all other techniques more\n",
    "    #ranks = [m2[x] for x in ranks]\n",
    "    return ranks\n",
    "\n",
    "dataset_to_compare = 'QUIRE'\n",
    "interval = 1-0.9\n",
    "\n",
    "print(\"all together\")\n",
    "a = df.loc[df['strategy']=='NN Single']['f1_auc'].to_numpy()\n",
    "b = df.loc[df['strategy']==dataset_to_compare]['f1_auc'].to_numpy()\n",
    "t, p = stats.ttest_ind(a,b)\n",
    "print(\"t = \" + str(t))\n",
    "print(\"p = \" + str(p))\n",
    "if p < interval:\n",
    "    print(\"Significant!\")\n",
    "else:\n",
    "    print(\"Not significant\")\n",
    "    \n",
    "a_s = []\n",
    "b_s = []\n",
    "    \n",
    "for dataset in df['dataset_id'].unique():\n",
    "    print(dataset_to_id[dataset])\n",
    "    a = df.loc[(df['dataset_id']==dataset) & (df['strategy']=='NN Single')]['f1_auc'].to_numpy()\n",
    "    b = df.loc[(df['dataset_id']==dataset) & (df['strategy']==dataset_to_compare)]['f1_auc'].to_numpy()\n",
    "    a_s.append(np.mean(a))\n",
    "    b_s.append(np.mean(b))\n",
    "    \n",
    "\n",
    "    ## Cross Checking with the internal scipy function\n",
    "    t, p = stats.ttest_ind(a,b)\n",
    "    #print(\"t = \" + str(t))\n",
    "    print(\"p = \" + str(p))\n",
    "    \n",
    "    if p < interval:\n",
    "        print(\"Significant!\")\n",
    "    else:\n",
    "        print(\"Not significant\")\n",
    "    ### You can see that after comparing the t statistic with the critical t value (computed internally) we get a good p value of 0.0005 and thus we reject the null hypothesis and thus it proves that the mean of the two distributions are different and statistically significant.\n",
    "\n",
    "    \n",
    "    #print(df.loc[(df['dataset_id']==dataset) & (df['strategy']=='NN Single')]['f1_auc'].mean())\n",
    "    #print(df.loc[(df['dataset_id']==dataset) & (df['strategy']=='MM')]['f1_auc'].mean())\n",
    "    print()\n",
    "\n",
    "print(\"Means\")\n",
    "    \n",
    "t, p = stats.ttest_ind(a_s,b_s)\n",
    "print(\"t = \" + str(t))\n",
    "print(\"p = \" + str(p))\n",
    "if p < interval:\n",
    "    print(\"Significant!\")\n",
    "else:\n",
    "    print(\"Not significant\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "code-oYwF_TsS",
   "language": "python",
   "name": "code-oywf_tss"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}