Prepare data for paper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "['MM' 'LC' 'Ent' 'Rand' 'GD' 'QUIRE' 'QBC' 'EER' 'NN Batch' 'NN Single1'\n",
      " 'NN Single1cos' 'LAL' 'SPAL' 'BMDR']\n",
      "{2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8, 10: 9, 11: 10, 12: 11, 32: 12, 33: 13, 13: 14}\n",
      "{3: 1, 4: 2, 2: 3, 8: 4, 10: 5, 11: 6, 12: 7, 13: 8, 14: 9, 15: 10, 16: 11, 5: 12, 27: 13, 29: 14, 6: 15}\n",
      "['MM' 'LC' 'Ent' 'Rand' 'GD' 'QUIRE' 'QBC' 'EER' 'NN Batch' 'NN Single'\n",
      " 'NN Single cos' 'LAL' 'SPAL' 'BMDR']\n",
      "['MM' 'LC' 'Ent' 'Rand' 'GD' 'QUIRE' 'QBC' 'EER' 'NN Batch' 'NN Single'\n",
      " 'LAL' 'SPAL' 'BMDR']\n",
      "[ 3  2  4  5  7  8  1  6 14 12 13  9 11 10]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>numqdata</th>\n",
       "      <th>round</th>\n",
       "      <th>init_labeled_size</th>\n",
       "      <th>num_of_queries</th>\n",
       "      <th>cost</th>\n",
       "      <th>performance</th>\n",
       "      <th>performance_plus_minus</th>\n",
       "      <th>dataset_id</th>\n",
       "      <th>strategy_id</th>\n",
       "      <th>dataset_random_seed</th>\n",
       "      <th>...</th>\n",
       "      <th>n_redundant</th>\n",
       "      <th>n_repeated</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>n_clusters_per_class</th>\n",
       "      <th>weights</th>\n",
       "      <th>flip_y</th>\n",
       "      <th>class_sep</th>\n",
       "      <th>hypercube</th>\n",
       "      <th>scale</th>\n",
       "      <th>random_state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.929415</td>\n",
       "      <td>0.014659</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.903619</td>\n",
       "      <td>0.085935</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.840795</td>\n",
       "      <td>0.169882</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.903605</td>\n",
       "      <td>0.041869</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.787143</td>\n",
       "      <td>0.217998</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58922</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0.934560</td>\n",
       "      <td>0.019395</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>96</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58923</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0.817690</td>\n",
       "      <td>0.040655</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>97</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58924</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0.947835</td>\n",
       "      <td>0.009473</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>95</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58925</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0.882504</td>\n",
       "      <td>0.075723</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>98</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58926</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0.964134</td>\n",
       "      <td>0.025045</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18026 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       numqdata  round  init_labeled_size  num_of_queries  cost  performance  \\\n",
       "2            48      0                  2              10     0     0.929415   \n",
       "3            48      0                  2              10     0     0.903619   \n",
       "4            48      0                  2              10     0     0.840795   \n",
       "5            48      0                  2              10     0     0.903605   \n",
       "6            48      0                  2              10     0     0.787143   \n",
       "...         ...    ...                ...             ...   ...          ...   \n",
       "58922        43      0                  7               9     0     0.934560   \n",
       "58923        43      0                  7               9     0     0.817690   \n",
       "58924        43      0                  7               9     0     0.947835   \n",
       "58925        43      0                  7               9     0     0.882504   \n",
       "58926        43      0                  7               9     0     0.964134   \n",
       "\n",
       "       performance_plus_minus  dataset_id  strategy_id  dataset_random_seed  \\\n",
       "2                    0.014659           1            3                    2   \n",
       "3                    0.085935           1            2                    9   \n",
       "4                    0.169882           1            3                    4   \n",
       "5                    0.041869           1            2                    7   \n",
       "6                    0.217998           1            2                    6   \n",
       "...                       ...         ...          ...                  ...   \n",
       "58922                0.019395          13            9                   96   \n",
       "58923                0.040655          13            9                   97   \n",
       "58924                0.009473          13            9                   95   \n",
       "58925                0.075723          13            9                   98   \n",
       "58926                0.025045          13            9                   99   \n",
       "\n",
       "       ... n_redundant  n_repeated  n_classes  n_clusters_per_class  weights  \\\n",
       "2      ...           0           0          2                     0        0   \n",
       "3      ...           0           0          2                     0        0   \n",
       "4      ...           0           0          2                     0        0   \n",
       "5      ...           0           0          2                     0        0   \n",
       "6      ...           0           0          2                     0        0   \n",
       "...    ...         ...         ...        ...                   ...      ...   \n",
       "58922  ...           0           0          7                     0        0   \n",
       "58923  ...           0           0          7                     0        0   \n",
       "58924  ...           0           0          7                     0        0   \n",
       "58925  ...           0           0          7                     0        0   \n",
       "58926  ...           0           0          7                     0        0   \n",
       "\n",
       "       flip_y  class_sep  hypercube  scale  random_state  \n",
       "2         0.0        0.0          0    0.0             2  \n",
       "3         0.0        0.0          0    0.0             9  \n",
       "4         0.0        0.0          0    0.0             4  \n",
       "5         0.0        0.0          0    0.0             7  \n",
       "6         0.0        0.0          0    0.0             6  \n",
       "...       ...        ...        ...    ...           ...  \n",
       "58922     0.0        0.0          0    0.0            96  \n",
       "58923     0.0        0.0          0    0.0            97  \n",
       "58924     0.0        0.0          0    0.0            95  \n",
       "58925     0.0        0.0          0    0.0            98  \n",
       "58926     0.0        0.0          0    0.0            99  \n",
       "\n",
       "[18026 rows x 26 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import argparse\n",
    "import csv\n",
    "\n",
    "import math\n",
    "import multiprocessing\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "from timeit import default_timer as timer\n",
    "from operator import itemgetter\n",
    "import dill\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import pairwise_distances\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.preprocessing import MinMaxScaler, RobustScaler\n",
    "\n",
    "from alipy.data_manipulate.al_split import split\n",
    "from alipy.experiment.al_experiment import AlExperiment\n",
    "from alipy.index import IndexCollection\n",
    "import timeit\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "font_size = 8\n",
    "\n",
    "tex_fonts = {\n",
    "    # Use LaTeX to write all text\n",
    "    \"text.usetex\": True,\n",
    "    \"font.family\": \"times\",\n",
    "    \"axes.labelsize\": font_size,\n",
    "    \"font.size\": font_size,\n",
    "    \"legend.fontsize\": font_size,\n",
    "    \"xtick.labelsize\": font_size,\n",
    "    \"ytick.labelsize\": font_size,\n",
    "    \"xtick.bottom\":True,\n",
    "    \"figure.autolayout\": True,\n",
    "    \n",
    "}\n",
    "\n",
    "sns.set_style('white')\n",
    "sns.set_context(\"paper\")\n",
    "plt.rcParams.update(tex_fonts)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# https://jwalton.info/Embed-Publication-Matplotlib-Latex/\n",
    "def set_size(width, fraction=1):\n",
    "    \"\"\"Set figure dimensions to avoid scaling in LaTeX.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    width: float\n",
    "            Document textwidth or columnwidth in pts\n",
    "    fraction: float, optional\n",
    "            Fraction of the width which you wish the figure to occupy\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    fig_dim: tuple\n",
    "            Dimensions of figure in inches\n",
    "    \"\"\"\n",
    "    # Width of figure (in pts)\n",
    "    fig_width_pt = width * fraction\n",
    "\n",
    "    # Convert from pt to inches\n",
    "    inches_per_pt = 1 / 72.27\n",
    "\n",
    "    # Golden ratio to set aesthetic figure height\n",
    "    # https://disq.us/p/2940ij3\n",
    "    golden_ratio = (5**.5 - 1) / 2\n",
    "\n",
    "    # Figure width in inches\n",
    "    fig_width_in = fig_width_pt * inches_per_pt\n",
    "    # Figure height in inches\n",
    "    fig_height_in = fig_width_in * golden_ratio\n",
    "\n",
    "    fig_dim = (fig_width_in, fig_height_in)\n",
    "\n",
    "    return fig_dim\n",
    "\n",
    "width = 505.89\n",
    "\n",
    "\n",
    "df = pd.read_csv(\"../datasets/ali/result.csv\")\n",
    "\n",
    "df['strategy'] = df['strategy'].map({\n",
    "    \"<class '__main__.Uncertainty'>{}\":\"MM1\", \n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True}\":\"ANNSingle10\", \n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True}\":\"ANNSingle1010\", \n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\": \"ANNSingle1010New\",\n",
    "    \"QueryInstanceUncertainty{'measure': 'margin'}\":\"MM2\", \n",
    "    \"QueryInstanceUncertainty{'measure': 'least_confident'}\":\"LC\", \n",
    "    \"QueryInstanceUncertainty{'measure': 'entropy'}\":\"Ent\", \n",
    "    \"QueryInstanceGraphDensity{}\":\"GD\", \n",
    "    \"QueryInstanceRandom{}\":\"Rand\", \n",
    "    \"QueryInstanceQUIRE{}\":\"QUIRE\", \n",
    "    \"QueryInstanceQBC{}\":\"QBC\", \n",
    "    \"QureyExpectedErrorReduction{}\":\"EER\", \n",
    "    \"QueryInstanceLAL{}\": \"LAL\",\n",
    "    \"QueryInstanceSPAL{}\": \"SPAL\",\n",
    "    \"QueryInstanceBMDR{}\": \"BMDR\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 200, 'TRUE_DISTANCES': True}\":\"ANNSingle200\", \n",
    "    \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/batch.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True}\":\"NN Batch\", \n",
    "    \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/batch.pickle', 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\": \"NN Batch\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10_cos_nrf.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': True}\": \"ANNSingleCosNrf\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10_cos.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\": \"ANNSingleCos\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10_nrf.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': True}\" : \"ANNSingleNrf\",\n",
    "    \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/batch_cos_nrf.pickle', 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': True}\": \"ANNBatchCosNrf\",\n",
    "    \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/batch_cos.pickle', 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\": \"NN Batchcos\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\": \"ANNSingle1010EuclidCos\",\n",
    "    \n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False, 'SLIDING_WINDOW': 5}\": \"NNeuc510\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False, 'SLIDING_WINDOW': 5}\": \"NNcos510\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 5, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False, 'SLIDING_WINDOW': 5}\": \"NNeuc55\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 5, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False, 'SLIDING_WINDOW': 5}\": \"NNcos55\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False, 'SLIDING_WINDOW': 1}\": \"NNeuc110\",\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 10, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False, 'SLIDING_WINDOW': 1}\": \"NNcos110\",\n",
    "    \n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Single1',\n",
    "    \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/single_10.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Single1cos',\n",
    "\n",
    "     \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/batch.pickle', 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False, 'INITIAL_BATCH_SAMPLING_ARG': 1, 'INITIAL_BATCH_SAMPLING_HYBRID_FURTHEST': 0, 'INITIAL_BATCH_SAMPLING_HYBRID_UNCERT': 0, 'INITIAL_BATCH_SAMPLING_METHOD': 'random'}\": 'NN Batch1',\n",
    "     \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '/lustre/ssd/ws/s5968580-IL_TD2/ali/batch.pickle', 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False, 'INITIAL_BATCH_SAMPLING_ARG': 1, 'INITIAL_BATCH_SAMPLING_HYBRID_FURTHEST': 0, 'INITIAL_BATCH_SAMPLING_HYBRID_UNCERT': 0, 'INITIAL_BATCH_SAMPLING_METHOD': 'random'}\": 'NN Batch1cos'\n",
    "})\n",
    "\n",
    "# drop variants\n",
    "df = df[df.strategy != 'MM1']\n",
    "#df = df[df.strategy != 'ANNSingle1010']\n",
    "df = df[df.strategy != 'ANNSingle200']\n",
    "df = df[df.strategy != 'ANNSingle10']\n",
    "df = df[df.strategy != 'ANNSingleCosNrf']\n",
    "df = df[df.strategy != 'ANNSingleNrf']\n",
    "df = df[df.strategy != 'ANNSingleCos']\n",
    "df = df[df.strategy != 'ANNBatchCosNrf']\n",
    "df = df[df.strategy != 'NN Batchcos']\n",
    "\n",
    "df = df[df.strategy != 'NNeuc510']\n",
    "df = df[df.strategy != 'NNcos510']\n",
    "df = df[df.strategy != 'NNeuc55']\n",
    "df = df[df.strategy != 'NNcos55']\n",
    "df = df[df.strategy != 'NNeuc110']\n",
    "df = df[df.strategy != 'NNcos110']\n",
    "\n",
    "#df = df[df.strategy != 'NN Single1']\n",
    "#df = df[df.strategy != 'NN Single1cos']\n",
    "df = df[df.strategy != 'NN Single']\n",
    "df = df[df.strategy != 'NN Batch1']\n",
    "df = df[df.strategy != 'NN Batch1cos']\n",
    "\n",
    "\n",
    "\n",
    "#for i in [13, 16, 21, 24,25,26]:\n",
    "for i in [22,23,19, 24,25,26]:\n",
    "    #for i in [22,23,19, 13,16,21]:\n",
    "    df = df[df.dataset_id != i]\n",
    "\n",
    "# it is the same as diabetes :(\n",
    "df = df[df.dataset_id != 9]\n",
    "\n",
    "\n",
    "df = df[df.dataset_id != 0] # synthetic\n",
    "df = df[df.dataset_id != 7] # ILPD\n",
    "#df = df[df.dataset_id != 5] # haberman\n",
    "df = df[df.dataset_id != 1] #BREAST\n",
    "df = df[df.dataset_id != 28] #parkinsons\n",
    "df = df[df.dataset_id != 18] # wine\n",
    "\n",
    "    \n",
    "#errors on this dataset\n",
    "df = df[df.dataset_id != 20]\n",
    "\n",
    "#toy dataset\n",
    "df = df[df.dataset_id != 21]\n",
    "\n",
    "# filter out partial results\n",
    "df = df[(df.strategy != 'SPAL') | (df.dataset_id != 12)]\n",
    "df = df[(df.strategy != 'LAL') | (df.dataset_id != 13)]\n",
    "\n",
    "\n",
    "print(len(df[(df.strategy == 'EER') & (df.dataset_id == 12)]))\n",
    "\n",
    "# filter out duplicates\n",
    "df = df.drop_duplicates(subset=['strategy_id', 'dataset_id', 'dataset_random_seed'], keep='last')\n",
    "\n",
    "df['strategy'] = df['strategy'].replace({\n",
    "    'MM1':'MM',\n",
    "    'MM2': 'MM',\n",
    "    'ANNSingle1010': 'NN Single',\n",
    "    'ANNSingle1010New': 'NN Single',\n",
    "})\n",
    "\n",
    "\n",
    "# remove NN Single\n",
    "df = df[(df.strategy != 'NN Single')]\n",
    "#df = df[(df.strategy != 'NN Single1')]\n",
    "#df = df[(df.strategy != 'NN Single1cos')]\n",
    "\n",
    "df = df [(df.strategy != 'ANNSingle1010EuclidCos')]\n",
    "print(df['strategy'].unique())\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# dasselbe für strategy_ids machen\n",
    "\n",
    "\n",
    "old_strategy_to_id= {\n",
    "    2: \"QBC\",\n",
    "    3: \"LC\",\n",
    "    4: \"MM\",\n",
    "    5: \"Ent\",\n",
    "    6: \"Rant\",\n",
    "    7: \"EER\",\n",
    "    8: \"GD\",\n",
    "    9: \"QUIRE\",\n",
    "    10: \"LAL\", \n",
    "    11: \"BMDR\",  \n",
    "    12: \"SPAL\",  \n",
    "    32: \"NN Single1\",\n",
    "    33: \"NN Single1cos\",\n",
    "    13: \"NN Batch\",\n",
    "    \n",
    "}\n",
    "\n",
    "old_strategy_to_id = {y:x for x,y in old_strategy_to_id.items()}\n",
    "\n",
    "new_strategy_to_id = {\n",
    "    1: \"QBC\",\n",
    "    2: \"LC\",\n",
    "    3: \"MM\",\n",
    "    4: \"Ent\",\n",
    "    5: \"Rant\",\n",
    "    6: \"EER\",\n",
    "    7: \"GD\",\n",
    "    8: \"QUIRE\",\n",
    "    9: \"LAL\", \n",
    "    10: \"BMDR\",  \n",
    "    11: \"SPAL\",  \n",
    "    12: \"NN Single1\",\n",
    "    13: \"NN Single1cos\",\n",
    "    14: \"NN Batch\",\n",
    "    \n",
    "}\n",
    "\n",
    "new_strategy_to_id = {y:x for x,y in new_strategy_to_id.items()}\n",
    "\n",
    "# old dataset_id -> new dataset_id\n",
    "mapping_dict = {}\n",
    "\n",
    "for dataset_name in new_strategy_to_id.keys():\n",
    "    mapping_dict[old_strategy_to_id[dataset_name]] = new_strategy_to_id[dataset_name]\n",
    "\n",
    "print(mapping_dict)\n",
    "\n",
    "df = df.replace({\"strategy_id\": mapping_dict})\n",
    "\n",
    "\n",
    "df['strategy'] = df['strategy'].replace({\n",
    "      'NN Single1': \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '../trained strategies/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\",\n",
    "      'NN Single1cos': \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '../trained strategies/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\",\n",
    "      'NN Batch': \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '../trained strategies/batch.pickle', 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\",\n",
    "        \n",
    "      \"MM\": \"QueryInstanceUncertainty{'measure': 'margin'}\", \n",
    "      \"LC\": \"QueryInstanceUncertainty{'measure': 'least_confident'}\", \n",
    "      \"Ent\": \"QueryInstanceUncertainty{'measure': 'entropy'}\", \n",
    "      \"GD\": \"QueryInstanceGraphDensity{}\", \n",
    "      \"Rand\": \"QueryInstanceRandom{}\", \n",
    "      \"QUIRE\": \"QueryInstanceQUIRE{}\", \n",
    "      \"QBC\": \"QueryInstanceQBC{}\", \n",
    "      \"EER\": \"QureyExpectedErrorReduction{}\", \n",
    "      \"LAL\": \"QueryInstanceLAL{}\",\n",
    "      \"SPAL\": \"QueryInstanceSPAL{}\",\n",
    "      \"BMDR\": \"QueryInstanceBMDR{}\",\n",
    "})\n",
    "\n",
    "\n",
    "old_dataset_to_id= {\n",
    "        0:\"synthetic\",                                                               \n",
    "        1:   \"breast\",                                                                            \n",
    "        2:   \"diabetes\",                                                                          \n",
    "        3:   \"fertility\",                                                                         \n",
    "        4:   \"german\",                                                                            \n",
    "        5:   \"haberman\",                                                                          \n",
    "        6:   \"heart\",                                                                             \n",
    "        7:   \"ILPD\",                                                                              \n",
    "        8:   \"ionosphere\",                                                                        \n",
    "        9:   \"pima\",                                                                              \n",
    "        10:   \"planning\",                                                                         \n",
    "        11:   \"australian\",                                                                       \n",
    "        12:   \"DWTC\",    \n",
    "        13: \"EMNIST\",\n",
    "        14: \"glass\",\n",
    "        15: \"olivetti\",\n",
    "        16: \"CIFAR-10\",\n",
    "        18: \"wine\",\n",
    "        19: \"adult50\",\n",
    "        20: \"abalone\",\n",
    "        21: \"adult1000\",\n",
    "        22: \"emnist50\",\n",
    "        23: \"cifar1050\",\n",
    "        24: \"adult100\",\n",
    "        25: \"emnist100\",\n",
    "        26: \"cifar10100\",\n",
    "        27: \"zoo\",\n",
    "        28: \"parkinsons\",\n",
    "        29: \"flags\",\n",
    "    \n",
    "}\n",
    "\n",
    "old_dataset_to_id = {y:x for x,y in old_dataset_to_id.items()}\n",
    "\n",
    "new_dataset_to_id = {\n",
    "    1: \"fertility\",\n",
    "    2: \"german\",\n",
    "    3: \"diabetes\",\n",
    "    4: \"ionosphere\",\n",
    "    5: \"planning\",\n",
    "    6: \"australian\",\n",
    "    7: \"DWTC\",\n",
    "    8: \"EMNIST\",\n",
    "    9: \"glass\",\n",
    "    10: \"olivetti\",\n",
    "    11: \"CIFAR-10\",\n",
    "    12: \"haberman\",\n",
    "    13: \"zoo\",\n",
    "    14: \"flags\",\n",
    "    15: \"heart\",\n",
    "}\n",
    "\n",
    "new_dataset_to_id = {y:x for x,y in new_dataset_to_id.items()}\n",
    "\n",
    "# old dataset_id -> new dataset_id\n",
    "mapping_dict = {}\n",
    "\n",
    "for dataset_name in new_dataset_to_id.keys():\n",
    "    mapping_dict[old_dataset_to_id[dataset_name]] = new_dataset_to_id[dataset_name]\n",
    "\n",
    "print(mapping_dict)\n",
    "\n",
    "df = df.replace({\"dataset_id\": mapping_dict})\n",
    "\n",
    "df = df.drop_duplicates(subset=['strategy', 'dataset_id', 'dataset_random_seed'], keep='last')\n",
    "\n",
    "\n",
    "# ab hier dann das csv was ich am Ende präsentiere\n",
    "df.to_csv('result.csv', index=False)\n",
    "\n",
    "\n",
    "df['strategy'] = df['strategy'].replace({\n",
    "        \"QueryInstanceUncertainty{'measure': 'margin'}\":\"MM\", \n",
    "    \"QueryInstanceUncertainty{'measure': 'least_confident'}\":\"LC\", \n",
    "    \"QueryInstanceUncertainty{'measure': 'entropy'}\":\"Ent\", \n",
    "    \"QueryInstanceGraphDensity{}\":\"GD\", \n",
    "    \"QueryInstanceRandom{}\":\"Rand\", \n",
    "    \"QueryInstanceQUIRE{}\":\"QUIRE\", \n",
    "    \"QueryInstanceQBC{}\":\"QBC\", \n",
    "    \"QureyExpectedErrorReduction{}\":\"EER\", \n",
    "    \"QueryInstanceLAL{}\": \"LAL\",\n",
    "    \"QueryInstanceSPAL{}\": \"SPAL\",\n",
    "    \"QueryInstanceBMDR{}\": \"BMDR\",\n",
    "    \n",
    "    \n",
    "      \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '../trained strategies/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Single',\n",
    "      \"<class '__main__.ANNQuerySingle'>{'NN_BINARY_PATH': '../trained strategies/single.pickle', 'HYPER_SAMPLING_SEARCH_ITERATIONS': 1, 'TRUE_DISTANCES': True, 'DISTANCE_METRIC': 'cosine', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Single cos',\n",
    "      \"<class '__main__.ANNQueryBatch'>{'NN_BINARY_PATH': '../trained strategies/batch.pickle', 'DISTANCE_METRIC': 'euclidean', 'STATE_INCLUDE_NR_FEATURES': False}\": 'NN Batch',\n",
    "})\n",
    "\n",
    "\n",
    "# rename back to <clas '__main' etc\n",
    "# rename then HERE to the names of the \n",
    "\n",
    "\n",
    "print(df['strategy'].unique())\n",
    "\n",
    "large_datasets_where_cosine_is_better_for_single = [7,8,10,11]\n",
    "\n",
    "for dataset_id in large_datasets_where_cosine_is_better_for_single:\n",
    "    df = df[(df.strategy != 'NN Single') | (df.dataset_id != dataset_id)]\n",
    "\n",
    "df = df[(df.strategy != 'NN Single cos') | (df.dataset_id.isin(large_datasets_where_cosine_is_better_for_single))]\n",
    "\n",
    "\n",
    "df['strategy'] = df['strategy'].replace({\n",
    "      'NN Single cos': 'NN Single',\n",
    "})\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# use cosine similarity for large datasets\n",
    "df = df.drop_duplicates(subset=['strategy', 'dataset_id', 'dataset_random_seed'], keep='last')\n",
    "\n",
    "print(df['strategy'].unique())\n",
    "print(df['strategy_id'].unique())\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "strategy \n",
       "Ent          1479\n",
       "GD           1479\n",
       "LC           1479\n",
       "MM           1479\n",
       "NN Batch     1479\n",
       "NN Single    1479\n",
       "QBC          1479\n",
       "Rand         1479\n",
       "LAL          1379\n",
       "EER          1279\n",
       "QUIRE        1279\n",
       "BMDR         1179\n",
       "SPAL         1078\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.value_counts(subset=['strategy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    strategy dataset_id       duration\n",
      "172     SPAL          1       6.327574\n",
      "173     SPAL          2     974.737428\n",
      "174     SPAL          3     350.478815\n",
      "175     SPAL          4      33.151179\n",
      "176     SPAL          5      13.336097\n",
      "177     SPAL          6     227.783949\n",
      "178     SPAL          9       8.664658\n",
      "179     SPAL         10      42.856895\n",
      "180     SPAL         13      10.738109\n",
      "181     SPAL         14      23.586630\n",
      "182     SPAL         15      25.107502\n",
      "187     SPAL         13  604800.000000\n",
      "189     SPAL         12  604800.000000\n",
      "193     SPAL         16  604800.000000\n",
      "      strategy dataset_id     duration\n",
      "114  NN Single          1     6.867454\n",
      "115  NN Single          2    28.636583\n",
      "116  NN Single          3    28.361288\n",
      "117  NN Single          4    20.524080\n",
      "118  NN Single          5    11.700790\n",
      "119  NN Single          6    28.397223\n",
      "120  NN Single          7    30.655202\n",
      "121  NN Single          8  7850.584493\n",
      "122  NN Single          9    12.809698\n",
      "123  NN Single         10    33.462379\n",
      "124  NN Single         11  5710.569733\n",
      "125  NN Single         12    18.255391\n",
      "126  NN Single         13     6.166812\n",
      "127  NN Single         14    11.460840\n",
      "128  NN Single         15    18.065725\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAACYCAYAAAAm509vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWsElEQVR4nO2dW2zk1nnH/yvtai/2cFQ7Rgt0KRcp4KmHVBAgaJ2lgLxUTThuHxoFDoVeDKnBeJDEWD10hMbF7iAQ1DbBPK2AotUOCqk14IhtbQRFOkMj7kOMLteO60ujoQC/ipuicGtHpOzdOI2hPijnmORwLhpxhhzO9wMMLw8vc4bQN4f8vv/5nzNHR0dHIAgitUzE3QGCIAYLBTlBpBwKcoJIORTkBJFyKMgJIuWcjbsDgyCXy8XdBYIYOm+//XZoeyqDHAD23niz7b4zk5OYuHB+iL0hiMHSaWBLbZC/9w8vtN33wJMLME0TAJDNZiEIAlZWViDLMorFIgD4tkVRHEqfCWIQDOWd3LIszM/PwzRNmKYJXddPfA3XdXH16tVI+vPiS99HNpuFoihoNpsAgO3tbaytrfGADm4TxKgylJFckiSIoghFUQAAy8vLKBQKEASh52sIgoDp6elI+nPlsc/iK1/7KmRZRrlchuM4aDabcF0XoihCEATftiRJkXwuQcTBULPrlmXBMAwoigJBEGDbNgzDQK1WAwAYhoGrV6/CNE3eZts2dF2HZVk4ODiIpB937txBuVxGJpPB5uYm/wFSVRW6rrdsE8QoM9QglyQJgiDwEVwURaiqCkEQePBPT09DURTYtg0AqFarKBQKkCQpspH89quvQlEUrK6uAoAvkNmPinebIEaZoSfeFEXBwsICNE2DYRhwHAfZbJbvz2QyA+/Dlcceg2EYEEURc3NzkGUZlmWh2WyiXC5DFEXfNkGMMkMJctu2+X+iKOLy5cswDAN3795FNpuF4zjY39/nj/De/1ZXV9FoNCCKIprNJr/Gacg/+ihm7/uMr02SJN+7d3CbIEaVM2mcaprL5ahOTowVuVxu/MQwk/ddirsLBJEIUhvkP3//J12POXP2HCYv3D+E3hBEfHQMctu2sby8jK2tLYiiCF3Xsb+/j8XFxdB2lq0GjsUrTGhy69YtrK6u8jZWL++EaZr8vH747+0/7XrMpS99E89/7zu8Fp7NZmHbNkzThCRJkGUZpmnCcRxomtZXPwgibjqW0ERRRLlcRqVSAQAUCgUsLi62bffSaDQgyzIURcHs7CyAY0FLLwEOoOfjTkNt6++hqipUVUWtVuMJvmKxiFqtBkEQIEkSXNcdeF8IYlB0rZMLggBN07g4pVs7Q5ZlLC0toVar8YA1TRPVarWt6MUwDL6fwerWuq5HHmzTntJds9mEpmkQBAGWZSGfz0f6WQQRFz2JYVRVxe7uboswpF07cFyC2t7ehiiKWFlZAfDx6BwmerEsCwB4O6NarUKWZYiiiEaj0cdXbM8TC7/P9fTeWn29Xu/7NYEgkkbPibf19XUsLS3hxo0bPbWbpsmlocDxO7pXqx4UvTDNODveyyDr1ZqmwXVdPnIbhoFSqRRJPZ4gkkDHILcsCzs7O5BlmT+ed2r3wh6/BUGA4zgQBAGmacK2bS5q8f4HALVajUtdXdflYhhd1/lnRRl4d+78GD/8zz04joNSqcRnyNXrdYiiiNXVVZimyZ9WKOiJUSQxYpharcbnclcqFaytrfV9rVwuB+v1V7oeRyU0Ii2MhBhGFEX+bhzF4/nZ+38pgl4RxOiTmCD3votHEeQffvBuX+dNTE7h3IXBT5IhiGGRmCCPGvM7T/Z1nvx7f43X37rVIoBxXZdbRkmSBMuyoOs6MpkMpqenUS6Xffvp/Z1ICrFYMtu2jYWFBV/pzTRNLC8v84Rbp/2DRMhkQgUwuq7zagFLEG5tbWFjY4ObS3j3E0RSiCXIRVHk88kZrIzFMuid9sfB7u4u/2zbtvkrBSsVBvcTRFKIbXEFURR5MISVp7rtTwKu65LklUg8sa6gwt5tmWHiSfcPk9nZWR7QrC+mafLRO2w/QSSBWINc07SORond9g8KrwDGdV3UajVomoZGowHDMHhCzmtdFbafIJJALNl15toqyzIyv0h0GYaBvb09nnjrtH/Q7+WapvkClZlNBoPXu91O+UcQcZMYxVuU5HI5/OgNs69zw+rklmWR3xuRaEZC8RY15+97MLJrUYATo0xqg/zu3f/t+9z3P/gQb7y+21YQw9q7CWROo78niKhIRJAzm6m1tTXfXHLLsmDbNgRB4O/qbBJLN/7xn/6g7/58+YnneB7AC3OKYe26rnOjiUqlAk3TsLW1BQA8+AkibmLNrjNEUfStlQYcj5q6rkNVVa4ki2qZpKjoJpAhiCSQiCAPo9FotLwLl0qlmHrTOySQIZJGIoO8XZDEJWltRzeBDEEkgUQGOXvcZb5v3vZh9iEoiAm2dxPIEEQSSESd3Jt4c10X9XodGxsbLYm3MP+3MHK5HJ7584f77s+Xn3gOly59oqWPJFclkkqnOnkigjxqcrkc3nzzVt/nT0xO4cJ5/yM3CWKIJDOWYpjgSHxaKMCJUSW1Qe7e68/+iXF24hwunacEGjH6pDbIv/29PzzV+V/73N/i5Tdb10ELqt4AcMto27ahKAqtn0Ykiliz65ZlYX5+nq9icvXq1cTUmDNCuA1UcH00wzD4gomKotD6aUTiiN00gindZFnG+vp6qHUSWw8tibAFIwzD4Ku4EkSSiL1O7jgOTNPE5uamTxvuXRDRsixeTksisiyTgSORWGIP8mw2C0VRMDMzA+D4nTe4IKKiKFzfnjSS2CeC8BJ7kDNYkoqp2oILIgLDd0HtVfXWbDZhGAafIefdTxBxE2t2nS12yAKbyUODCyI6joODg4OhJ7PCbKDC2sNsoSizTiSF1CreXnvrdDr3sDo5qd6IpDKWijfhYnT2TwwKcGIUSW2Qv3svmkf7qcmzOPrpz0MFLsytxjRNXg5cWVmBLMsoFos8KccsnWkKKhEHiQhyy7LQbDZ5UFiWhWKxCMuysLKywr3SWHsv/PGL34qkb89+4Rt4MGD7xGg0GigUCigWi3wW3fb2ti+YWV6BIOIi9iB3XRebm5vY2NjgbbduHc8g84plgONyVbVaxerqaix9DcJGdcuykM/nAQDNZpOv+CJJEk1RJWIn9iBvNBqYm5vztbWzeRJFEXt7e8Po1omo1+v8h4cFdKVS4f50QfMLghgmsdfJXdf1OakkWdkWhmEYKJVKLdJb27aRzWZ5zZzcW4m4iD3IVVX1BYAgCNjc3Aw9Ns6FD8OEMaZpQtd1XLt2DTs7OygUCtx7vVwuQ5IkyLIM4Fi+SxBxkIg6uWmaPIAdx+FKMq8tFNB74i2Xy+HXv/2lSPr27Be+gQcvtmbF6V2bSBJjaf9kvvVaJNeamjyLzNSllnYSxhBJYizFMGGjb5RQgBOjQmqD/N17H8TdhaEzNTmJzNSFuLtBJIzUBvmT9efi7sLQ+atPfQ7f/LNnWhR3YZZVQKsSj5R56aRrdt22bczPz/Oylq7rqFarbdvDzmX2TsH9vcLMI4jubG9vY21tzZcUDLOkCirxSJmXXroGuSiKKJfLqFQqAIBCoYDFxcW27cFz8/k8FEWBoig4PDzsy96JlaGI7rC57d0EOMHqAFUL0ktPdXJBEKBpWsto2q7dy+HhISzL4oaHoihyTzR2XtDeqVarQdd13x8qE5TQiN6eX718ma8A2+lHM7jqKq3Cmm56FsOoqhrqdtKunZHJZLiTKTN+EEURqqpCEAQYhuGzd6pWq3zk9+K1gyLC+e4/P8//3ek+BZV4pMxLNydKvK2vr2NpaQk3btzoqd2LIAiYmZlBo9FANpsNXRiQvReKotiS/AmzgyL8/PbnP89n9JXLZb7Ge7FY9Cn2vO/njuNAVVXfNpEuuga5ZVnY2dmBLMv88bxTuxfbtrG3t8dHh/39fZRKJT56OI6D/f19qKrKR/lSqYRGo8FHdlEUW+yg6P0xnIyQwa/98q/4avjtLKsEQfDN/AtuE+khxYq3N+LuxtAJq5OTMm88GFPF231xdyERUIATqQ3yd+/di7sLiefll/4N//LC88hkMpienuYTgWzbbrGxYq9c2WyWV0iY7RVl5pPNUIOclckEQeAzzQC0zDa7desWN2HwnsM81XqZibb0r0bXY8adZz75MLa2tgCgJavutbHyrvem6zqazabP9oqCPNkMLchZppcFsuu6qFQqXJ3FRDPA8R8VM13wngOgb9Uc0Uru0UcBhNfJvTZWiqJgaWkJsiyjXC7z4PfaXhHJZWimEY1Gw/eHJAiCb4HAMNFMo9FoeadsZw1F9Ifrui2LVrDAZqIa27ZRLpeRyWR8hh5e2ysiuQwtyLutfhImmgmDJk9Ei2maLfc0aGPFRnpvQHttr4hkM7QgVxSFv/exAA6KYQC/aCbMBJEUWdHiFSUxW6ugjZWiKFwPPzc312J7RSSbodbJDcPA7u4uZmZmuDCGJeGCiTe2L5h4Y+KOTuRyOTzyF9H4rqeZ7d9V8eDFi742EhqNJom0f6rVagMrvxyLYd6K/LppY2piEpnzU742Es+MJokUwxwcHKBWqw2s/BIcoYjeoABPH7EF+aCzsu/d+9lAr08ki3OTE8hMpVbbdSqGfle879/BUZzNM2dil07HdqNYfyuqLhMjQO3xTx///xcWVsDHSVq23BbDq95jAh9Zln0uR0EV4Cgz9MUVWA08LGhZuaaXYwkiiNfCStd1Xuv3Go0YhoFsNgtFUbiLDlPzMV+Dra0tbGxs9JTkHQViX0ElCNNFE8RJ8VYGdnd3ef3f+/ekKAquX7+OSqWCQqEA0zT5E2Sz2eQ5iTS55cQe5KxmblkWZFmGqqpta6/dBDXE+PLDV273FJRh6j32d8dG/DAV4CgTe5Czx3PbtrnM9fbt2x2PJYggQQur2dlZHqjB93Gvei9MExCmAhxlYkm8sXdv13VRr9chCAJM0+RJDjYxhXm6eY9Ny3sSES25R/M4+38/BQDuL8+sxjRN4xOkmHpPFEXMzc1BlmU0Gg3f7EbHcVIlCEqtM4z8l8/G3Q1iiNQe/zQeuDjV8Zg0q/kSKYYZNKykQowH5ya7v3mm6T37JKQ2yKcwGXcXiGHyEfD+vY86HvLwJ3+j6zFeJifP4OJU7GmrUxNZkHdzfblx4wYXHriuC0VRsLKywm2cV1ZWfPZDmqbBsixf+87ODtbX13tKitw0/ieqr0aMKU+pD7XYXoWtKRdmlwX415YLCnCGSSQ/UyypoaoqFEVBoVDgyycxQQv7YoVCAaqqchWSJEn832xRBcMw4Lqur12WZayvr1MNnRgaL33/RZ9wJmxNOUZwDTqvMCcowBk2kYzk3VxfeoU9DSiKwkdrx3Fgmib3faMJFMSweOyxK/j6V7/Cba864bXLkiTJl+QL2mcNm8hG8iiQJAmCIPgex9kv4MzMTKSfRRDduPPjO6G2V0GCdllBtVw7+6xhEUmQd3N98S5x1K2MoShK6GJ9wUkHBDFomIqu24zJoF1WUJgTZp81TCJ5XGfi/mq1yl1fvOuira6uQtd1X3CzR3PLsniyjv0AXL58GYZh4OGHH/ZNWjEMI3Q5JoIYBL/12Ss+4QwA35pyTLTFksRsDbrgWnNBAc6wGYgYZpCuL72Qy+VQ+tYPYvlsIj08pT6E+y92LsUmRWAzdDHMoF1feuEp9aHYPptIB5OTZ7oeMwo5otTKWtv9qhFEGhlLWeuHd3tXNhHESZmYPINz50dDDTf0tdDCFGxMMdSt3auk68btF94bzJcgCADS75zF62++ytVvruuG2k15VW+MTnZT5XI5VFV3Gob6U9ROwdZLu1cJRxBxI2T86rcwuymv6o3RzW6qk6quX4b+vMEUbJubm/wLdWoHwNdI8yrhCCJJhNlNhWXe47CbGnqQt1OwdVK2hSnhCCLJdAraYdtNxZY5aKdga9feTglHEEkgaDcVVL0x4rCbGmrizWv9BHysYOvWHlTC0WM7kQS86reg3VRQ9cZmarLjOtlNea8bhdAmtXXyzev/Hnc3iBRzZeEBnL90MmOSQarjxrJOfmXhgbi7QKSYiR7UcEHiqgylNsjPkhaGGCQfHeGjn/n/yM6cPYOJi+3TXHF5IaQ2yN/5m3fi7gIxZlz4owt49Qd+gQzLNUmSBFmWQ4UuTEjjPY+ZUEiShGw267vOSUttkWXXbdvG/Pw8nz9brVZ5+8LCgk8UYJomlpeX4bouLMvyncfKCsF2yqwTSScokGk0GhBFEcViEbVara3QJdjO3t0lScLOzk7LdU5KZEEuiiLy+TxXpx0eHvLOapoGwzD4sa7rIp/P8y/nVbWpqopqtUpqN2LkYXJWy7KQz+d7Po+N3oZhYHFxse/rMCKtkx8eHnJ1GjNwBPyLGHbLMIqiiL29Pb5Najdi1KnX6yd2hWEDoHdw7Oc6QMRBnslk+DvDwcFBi2rNsixudtcrpHYjRhnDMFAqlU7kMsxeTb22av1chzEQxZsgCJiZmUGj0eBtmqb19F4d9iNAajdiVPAKWVgu6dq1a3ylXu9+13X5O7a3XZZlWJaFWq2GYrEYep2TEFl23bZt7O3t8V+e/f19lEol/rgtyzIf6Q3DwN7eHlzXheM4PrWbZVlYW1sjtRsxkmiaxjPnLKfUbj8AvoBnsB3wl9xOM3kltYq3vf/Y634gQURItzp5GJZlRVI/H0vF22SG1kIjks8wBDKpDfKP3v8w7i4QY8rh3ffxyhuvdRTFMItyr4uMYRgtLjHB8/oRxQz0cZ0pdwDwZY6CFlCWZfHZOGwyvXd7eXkZa2trJ3onyeVyePnrfxfxtyGI3vjEn/wm/usn7/C/ZV3XUSgUIAgClpeXkc/nsbi4CFEUUalUfLHgNZGwbdt3nqqqvu2trS3+mbE9rjcaDd4pVk7zilyA418v9uXYrxQLclZrj9PamSBOC0uoMTHL9PQ03+ddM7CdSww7L3idXhlokMuyjKWlJRQKhVBTOtd1cXBw4HsvYcKZJBjWE0SUMDGL67pcqupdTgwId4kJimBOKooZqDOMJEnY3t6GKIpYWVnh7Y7jwLIsOI6DjY0NAMe/TswWp10tkGStxKgSFLNomgZZlltG5KBLTPC8fkQxAx3J2WMHqwW6rgtBEEIXYrdtG47jAABu377d9nrsWgSRZLziFtu2oes66vU6RFHE448/zv/eS6USd40pFos+lxgmgmHnzc3N+bZ7Hc0HGuQskSYIAhzH8S1sGEwymKbJExBsITlFUfh7uuu6qNfrFOTESNBNFBMc5LyiGAabnOWln/xUesUwr/8o7m4QY8qZsxOYuHDuROecVhQzlmKY/Gc+FXcXCCIZHKWYL37xi223vf9+5JFHBv7ZUZzTaX+7fZ3uQVgb3Zfu20m4Lyc5fjRWbCMIom9SHeTB2rx3O6rF5Hr97CjO6bS/3b5O9yCsje5Lb9tRc9Lrn+T4VCbeTgqtZx4O3ZdwRu2+pHok75Wnn3467i4kErov4YzafaGRnCBSDo3kBJFyKMgJIuWkVgzDCK5OEWwHPva57rSdlllxYat1hH3Pcbg/3e5Fu+84cvfmRBX4EWV/f//o5s2bvrabN28eOY5zdHR0dHT9+vWu22nCez/afc9xuT+d7kVa7s3YPq7v7u7yKX22bXfdTivtvuc43p9ev+Oo3ZuxDXKCGBfGNshnZ2e5CYUoil2300q77zmO96fX7zhq92Ys6uS6rnMjSTZXXdM0NBoNZLNZ7iXXaTuutaUHQfB+BL/3ON2fbvciDfdmLII8CHnIdYbuT3tG8d6M5eM6ecV1hu5Pe0bx3ozlSE4Q48RYjuQEMU5QkBNEyqEgJ4iUQ0FOECmHgpwgUs7/A1TdfJkSIRQRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x155.745 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def show_values_on_bars(axs, h_v=\"v\", space=0.4):\n",
    "    def _show_on_single_plot(ax):\n",
    "        if h_v == \"v\":\n",
    "            for p in ax.patches:\n",
    "                _x = p.get_x() + p.get_width() / 2\n",
    "                _y = p.get_y() + p.get_height()\n",
    "                value = int(p.get_height())\n",
    "                ax.text(_x, _y, \"{:,.0f}\".format(value), ha=\"center\") \n",
    "        elif h_v == \"h\":\n",
    "            for p in ax.patches:\n",
    "                _x = p.get_x() + p.get_width() + float(space)\n",
    "                _y = p.get_y() + p.get_height() - 0.15\n",
    "                value = int(p.get_width())\n",
    "                ax.text(_x, _y, \"{:,.0f}\".format(value), ha=\"left\")\n",
    "\n",
    "    if isinstance(axs, np.ndarray):\n",
    "        for idx, ax in np.ndenumerate(axs):\n",
    "            _show_on_single_plot(ax)\n",
    "    else:\n",
    "        _show_on_single_plot(axs)\n",
    "\n",
    "\n",
    "plt.figure(figsize=set_size(width, fraction=0.5))\n",
    "from numpy import median\n",
    "from matplotlib.ticker import ScalarFormatter,AutoMinorLocator\n",
    "\n",
    "\n",
    "df_dur = df.groupby(['strategy', 'dataset_id'])['duration'].mean().to_frame()\n",
    "df_dur = df_dur.reset_index()  \n",
    "\n",
    "#print(df_dur.loc[df_dur['strategy']=='Rand'])\n",
    "#print(df_dur.loc[df_dur['strategy']=='NN Single'])\n",
    "#print(df_dur.loc[df_dur['strategy']=='MM'])\n",
    "\n",
    "df_dur2 =  df.groupby(['strategy'])['duration'].mean().to_frame()\n",
    "\n",
    "timout_duration = 604800\n",
    "#timout_duration = 259200\n",
    "\n",
    "# timeout\n",
    "df_dur = df_dur.append({'strategy':'LAL', 'dataset_id':'13', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'EER', 'dataset_id':'13', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'QUIRE', 'dataset_id':'13', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'BMDR', 'dataset_id':'13', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'SPAL', 'dataset_id':'13', 'duration':timout_duration}, ignore_index=True)\n",
    "\n",
    "df_dur = df_dur.append({'strategy':'BMDR', 'dataset_id':'12', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'SPAL', 'dataset_id':'12', 'duration':timout_duration}, ignore_index=True)\n",
    "\n",
    "\n",
    "df_dur = df_dur.append({'strategy':'EER', 'dataset_id':'16', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'QUIRE', 'dataset_id':'16', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'BMDR', 'dataset_id':'16', 'duration':timout_duration}, ignore_index=True)\n",
    "df_dur = df_dur.append({'strategy':'SPAL', 'dataset_id':'16', 'duration':timout_duration}, ignore_index=True)\n",
    "\n",
    "\n",
    "print(df_dur[df_dur['strategy'] == 'SPAL'])\n",
    "print(df_dur[df_dur['strategy'] == 'NN Single'])\n",
    "\n",
    "#print(df_dur.loc[df_dur['strategy'] =='LAL']['duration'].mean())\n",
    "\n",
    "#print(df_dur.loc[df_dur['strategy'] =='Rand']['duration'].mean())\n",
    "\n",
    "df_hist_new = pd.DataFrame(columns=['strategy', 'duration'])\n",
    "for strategy in df.strategy.unique():\n",
    "    df_hist_new = df_hist_new.append({\n",
    "        'strategy':strategy, \n",
    "        'duration': df_dur.loc[df_dur['strategy'] ==strategy]['duration'].mean()\n",
    "    }, ignore_index=True)\n",
    "\n",
    "#print(df_hist_new)\n",
    "\n",
    "df_hist_new = df_hist_new.sort_values(by=['duration'])\n",
    "g = sns.barplot(data=df_hist_new, x=\"duration\", y=\"strategy\")#, ci=68, capsize=.3)#, fill=True, common_norm=False, alpha=.5, linewidth=0)\n",
    "show_values_on_bars(g, \"h\", 10.4)\n",
    "g.set_xscale(\"log\")\n",
    "#g.xaxis.set_minor_locator(AutoMinorLocator(5))\n",
    "\n",
    "#print(g.get_xlim())\n",
    "g.set_xlim(g.get_xlim()[0],g.get_xlim()[1]+170000)\n",
    "\n",
    "import matplotlib.ticker\n",
    "g.get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())\n",
    "g.get_xaxis().set_minor_formatter(matplotlib.ticker.NullFormatter())\n",
    "g.get_xaxis().set_major_formatter(\n",
    "    matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))\n",
    "\n",
    "g.xaxis.tick_bottom()\n",
    "g.set_xlabel(\"\")\n",
    "#g.set_xlabel(\"duration in s (walltime)\")\n",
    "g.set_ylabel(\"\")\n",
    "g.get_figure().savefig('../paper/figures/performance.pdf', format='pdf', bbox_inches='tight')\n",
    "g\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>strategy  </th><th>NN Single  </th><th>NN Batch  </th><th>MM       </th><th>QBC       </th><th>LC        </th><th>GD        </th><th>BMDR      </th><th>Ent       </th><th>LAL       </th><th>SPAL      </th><th>EER       </th><th>QUIRE     </th><th>Rand      </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>australian</td><td>85.4% (0)  </td><td>85.3% (4) </td><td>85.4% (2)</td><td>85.3% (3) </td><td>85.4% (1) </td><td>85.2% (5) </td><td>85.1% (7) </td><td>85.4% (2) </td><td>85.2% (6) </td><td>84.3% (10)</td><td>84.6% (9) </td><td>76.2% (11)</td><td>84.9% (8) </td></tr>\n",
       "<tr><td>CIFAR-10  </td><td>37.0% (0)  </td><td>36.9% (1) </td><td>36.9% (2)</td><td>36.0% (5) </td><td>35.9% (6) </td><td>36.3% (4) </td><td>0.0% (9)  </td><td>35.7% (7) </td><td>34.9% (8) </td><td>0.0% (9)  </td><td>0.0% (9)  </td><td>0.0% (9)  </td><td>36.8% (3) </td></tr>\n",
       "<tr><td>diabetes  </td><td>74.3% (4)  </td><td>74.5% (2) </td><td>74.3% (5)</td><td>74.4% (3) </td><td>74.3% (6) </td><td>75.0% (0) </td><td>74.7% (1) </td><td>74.3% (5) </td><td>74.3% (7) </td><td>73.9% (10)</td><td>74.2% (9) </td><td>66.5% (11)</td><td>74.2% (8) </td></tr>\n",
       "<tr><td>DWTC      </td><td>74.9% (0)  </td><td>74.1% (2) </td><td>74.7% (1)</td><td>71.7% (6) </td><td>71.0% (7) </td><td>65.1% (9) </td><td>0.0% (11) </td><td>68.5% (8) </td><td>72.2% (5) </td><td>0.0% (11) </td><td>72.8% (4) </td><td>53.4% (10)</td><td>72.9% (3) </td></tr>\n",
       "<tr><td>EMNIST    </td><td>69.1% (1)  </td><td>68.6% (2) </td><td>69.5% (0)</td><td>63.4% (4) </td><td>61.6% (5) </td><td>43.8% (7) </td><td>0.0% (8)  </td><td>59.2% (6) </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>66.1% (3) </td></tr>\n",
       "<tr><td>fertility </td><td>88.5% (3)  </td><td>88.3% (4) </td><td>88.0% (8)</td><td>88.2% (6) </td><td>88.0% (8) </td><td>88.3% (5) </td><td>88.6% (2) </td><td>88.0% (9) </td><td>88.0% (7) </td><td>88.6% (1) </td><td>86.6% (11)</td><td>89.1% (0) </td><td>87.4% (10)</td></tr>\n",
       "<tr><td>flags     </td><td>57.7% (0)  </td><td>56.4% (5) </td><td>56.6% (4)</td><td>55.9% (9) </td><td>55.4% (10)</td><td>57.0% (2) </td><td>57.4% (1) </td><td>54.7% (11)</td><td>55.9% (8) </td><td>56.9% (3) </td><td>56.0% (7) </td><td>50.4% (12)</td><td>56.2% (6) </td></tr>\n",
       "<tr><td>german    </td><td>75.7% (2)  </td><td>75.6% (3) </td><td>75.4% (9)</td><td>75.5% (6) </td><td>75.4% (10)</td><td>76.2% (1) </td><td>75.6% (5) </td><td>75.4% (11)</td><td>75.5% (7) </td><td>76.5% (0) </td><td>75.4% (8) </td><td>70.9% (12)</td><td>75.6% (4) </td></tr>\n",
       "<tr><td>glass     </td><td>69.0% (0)  </td><td>68.1% (2) </td><td>68.6% (1)</td><td>67.5% (6) </td><td>68.0% (3) </td><td>66.4% (8) </td><td>67.6% (5) </td><td>66.3% (9) </td><td>66.0% (10)</td><td>61.7% (11)</td><td>67.2% (7) </td><td>48.2% (12)</td><td>67.6% (4) </td></tr>\n",
       "<tr><td>haberman  </td><td>72.9% (4)  </td><td>72.3% (5) </td><td>72.9% (3)</td><td>73.3% (0) </td><td>72.9% (1) </td><td>71.6% (8) </td><td>71.6% (9) </td><td>72.9% (2) </td><td>72.3% (6) </td><td>0.0% (12) </td><td>72.0% (7) </td><td>63.2% (11)</td><td>71.5% (10)</td></tr>\n",
       "<tr><td>heart     </td><td>79.4% (0)  </td><td>79.4% (1) </td><td>79.1% (7)</td><td>79.1% (9) </td><td>79.1% (6) </td><td>79.2% (4) </td><td>79.3% (2) </td><td>79.1% (7) </td><td>79.3% (3) </td><td>78.6% (10)</td><td>79.1% (5) </td><td>74.7% (11)</td><td>79.1% (8) </td></tr>\n",
       "<tr><td>ionosphere</td><td>90.3% (0)  </td><td>89.3% (9) </td><td>90.0% (2)</td><td>89.9% (4) </td><td>90.0% (1) </td><td>89.6% (6) </td><td>89.8% (5) </td><td>90.0% (1) </td><td>89.9% (3) </td><td>83.2% (10)</td><td>89.4% (8) </td><td>53.7% (11)</td><td>89.6% (7) </td></tr>\n",
       "<tr><td>olivetti  </td><td>72.2% (1)  </td><td>70.4% (5) </td><td>72.3% (0)</td><td>72.1% (3) </td><td>72.1% (2) </td><td>65.5% (10)</td><td>64.3% (11)</td><td>70.7% (4) </td><td>65.6% (9) </td><td>66.5% (6) </td><td>65.7% (8) </td><td>51.2% (12)</td><td>66.1% (7) </td></tr>\n",
       "<tr><td>planning  </td><td>72.3% (3)  </td><td>74.0% (0) </td><td>72.1% (4)</td><td>72.7% (2) </td><td>72.1% (4) </td><td>69.6% (8) </td><td>68.9% (9) </td><td>72.1% (5) </td><td>71.5% (7) </td><td>73.6% (1) </td><td>68.2% (11)</td><td>71.8% (6) </td><td>68.4% (10)</td></tr>\n",
       "<tr><td>zoo       </td><td>93.4% (1)  </td><td>93.5% (0) </td><td>93.0% (4)</td><td>92.5% (10)</td><td>92.9% (7) </td><td>92.7% (9) </td><td>93.3% (2) </td><td>92.8% (8) </td><td>92.3% (11)</td><td>92.2% (12)</td><td>92.9% (6) </td><td>93.2% (3) </td><td>93.0% (5) </td></tr>\n",
       "<tr><td>mean %    </td><td>74.2% (0)  </td><td>73.8% (2) </td><td>73.9% (1)</td><td>73.2% (3) </td><td>72.9% (4) </td><td>70.8% (7) </td><td>61.1% (10)</td><td>72.3% (6) </td><td>68.2% (8) </td><td>55.7% (12)</td><td>65.6% (9) </td><td>57.5% (11)</td><td>72.6% (5) </td></tr>\n",
       "<tr><td>mean (r)  </td><td>1.27       </td><td>3.00      </td><td>3.47     </td><td>5.07      </td><td>5.13      </td><td>5.73      </td><td>5.80      </td><td>6.33      </td><td>7.00      </td><td>7.60      </td><td>7.80      </td><td>9.27      </td><td>6.40      </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "'<table>\\n<thead>\\n<tr><th>strategy  </th><th>NN Single  </th><th>NN Batch  </th><th>MM       </th><th>QBC       </th><th>LC        </th><th>GD        </th><th>BMDR      </th><th>Ent       </th><th>LAL       </th><th>SPAL      </th><th>EER       </th><th>QUIRE     </th><th>Rand      </th></tr>\\n</thead>\\n<tbody>\\n<tr><td>australian</td><td>85.4% (0)  </td><td>85.3% (4) </td><td>85.4% (2)</td><td>85.3% (3) </td><td>85.4% (1) </td><td>85.2% (5) </td><td>85.1% (7) </td><td>85.4% (2) </td><td>85.2% (6) </td><td>84.3% (10)</td><td>84.6% (9) </td><td>76.2% (11)</td><td>84.9% (8) </td></tr>\\n<tr><td>CIFAR-10  </td><td>37.0% (0)  </td><td>36.9% (1) </td><td>36.9% (2)</td><td>36.0% (5) </td><td>35.9% (6) </td><td>36.3% (4) </td><td>0.0% (9)  </td><td>35.7% (7) </td><td>34.9% (8) </td><td>0.0% (9)  </td><td>0.0% (9)  </td><td>0.0% (9)  </td><td>36.8% (3) </td></tr>\\n<tr><td>diabetes  </td><td>74.3% (4)  </td><td>74.5% (2) </td><td>74.3% (5)</td><td>74.4% (3) </td><td>74.3% (6) </td><td>75.0% (0) </td><td>74.7% (1) </td><td>74.3% (5) </td><td>74.3% (7) </td><td>73.9% (10)</td><td>74.2% (9) </td><td>66.5% (11)</td><td>74.2% (8) </td></tr>\\n<tr><td>DWTC      </td><td>74.9% (0)  </td><td>74.1% (2) </td><td>74.7% (1)</td><td>71.7% (6) </td><td>71.0% (7) </td><td>65.1% (9) </td><td>0.0% (11) </td><td>68.5% (8) </td><td>72.2% (5) </td><td>0.0% (11) </td><td>72.8% (4) </td><td>53.4% (10)</td><td>72.9% (3) </td></tr>\\n<tr><td>EMNIST    </td><td>69.1% (1)  </td><td>68.6% (2) </td><td>69.5% (0)</td><td>63.4% (4) </td><td>61.6% (5) </td><td>43.8% (7) </td><td>0.0% (8)  </td><td>59.2% (6) </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>0.0% (8)  </td><td>66.1% (3) </td></tr>\\n<tr><td>fertility </td><td>88.5% (3)  </td><td>88.3% (4) </td><td>88.0% (8)</td><td>88.2% (6) </td><td>88.0% (8) </td><td>88.3% (5) </td><td>88.6% (2) </td><td>88.0% (9) </td><td>88.0% (7) </td><td>88.6% (1) </td><td>86.6% (11)</td><td>89.1% (0) </td><td>87.4% (10)</td></tr>\\n<tr><td>flags     </td><td>57.7% (0)  </td><td>56.4% (5) </td><td>56.6% (4)</td><td>55.9% (9) </td><td>55.4% (10)</td><td>57.0% (2) </td><td>57.4% (1) </td><td>54.7% (11)</td><td>55.9% (8) </td><td>56.9% (3) </td><td>56.0% (7) </td><td>50.4% (12)</td><td>56.2% (6) </td></tr>\\n<tr><td>german    </td><td>75.7% (2)  </td><td>75.6% (3) </td><td>75.4% (9)</td><td>75.5% (6) </td><td>75.4% (10)</td><td>76.2% (1) </td><td>75.6% (5) </td><td>75.4% (11)</td><td>75.5% (7) </td><td>76.5% (0) </td><td>75.4% (8) </td><td>70.9% (12)</td><td>75.6% (4) </td></tr>\\n<tr><td>glass     </td><td>69.0% (0)  </td><td>68.1% (2) </td><td>68.6% (1)</td><td>67.5% (6) </td><td>68.0% (3) </td><td>66.4% (8) </td><td>67.6% (5) </td><td>66.3% (9) </td><td>66.0% (10)</td><td>61.7% (11)</td><td>67.2% (7) </td><td>48.2% (12)</td><td>67.6% (4) </td></tr>\\n<tr><td>haberman  </td><td>72.9% (4)  </td><td>72.3% (5) </td><td>72.9% (3)</td><td>73.3% (0) </td><td>72.9% (1) </td><td>71.6% (8) </td><td>71.6% (9) </td><td>72.9% (2) </td><td>72.3% (6) </td><td>0.0% (12) </td><td>72.0% (7) </td><td>63.2% (11)</td><td>71.5% (10)</td></tr>\\n<tr><td>heart     </td><td>79.4% (0)  </td><td>79.4% (1) </td><td>79.1% (7)</td><td>79.1% (9) </td><td>79.1% (6) </td><td>79.2% (4) </td><td>79.3% (2) </td><td>79.1% (7) </td><td>79.3% (3) </td><td>78.6% (10)</td><td>79.1% (5) </td><td>74.7% (11)</td><td>79.1% (8) </td></tr>\\n<tr><td>ionosphere</td><td>90.3% (0)  </td><td>89.3% (9) </td><td>90.0% (2)</td><td>89.9% (4) </td><td>90.0% (1) </td><td>89.6% (6) </td><td>89.8% (5) </td><td>90.0% (1) </td><td>89.9% (3) </td><td>83.2% (10)</td><td>89.4% (8) </td><td>53.7% (11)</td><td>89.6% (7) </td></tr>\\n<tr><td>olivetti  </td><td>72.2% (1)  </td><td>70.4% (5) </td><td>72.3% (0)</td><td>72.1% (3) </td><td>72.1% (2) </td><td>65.5% (10)</td><td>64.3% (11)</td><td>70.7% (4) </td><td>65.6% (9) </td><td>66.5% (6) </td><td>65.7% (8) </td><td>51.2% (12)</td><td>66.1% (7) </td></tr>\\n<tr><td>planning  </td><td>72.3% (3)  </td><td>74.0% (0) </td><td>72.1% (4)</td><td>72.7% (2) </td><td>72.1% (4) </td><td>69.6% (8) </td><td>68.9% (9) </td><td>72.1% (5) </td><td>71.5% (7) </td><td>73.6% (1) </td><td>68.2% (11)</td><td>71.8% (6) </td><td>68.4% (10)</td></tr>\\n<tr><td>zoo       </td><td>93.4% (1)  </td><td>93.5% (0) </td><td>93.0% (4)</td><td>92.5% (10)</td><td>92.9% (7) </td><td>92.7% (9) </td><td>93.3% (2) </td><td>92.8% (8) </td><td>92.3% (11)</td><td>92.2% (12)</td><td>92.9% (6) </td><td>93.2% (3) </td><td>93.0% (5) </td></tr>\\n<tr><td>mean %    </td><td>74.2% (0)  </td><td>73.8% (2) </td><td>73.9% (1)</td><td>73.2% (3) </td><td>72.9% (4) </td><td>70.8% (7) </td><td>61.1% (10)</td><td>72.3% (6) </td><td>68.2% (8) </td><td>55.7% (12)</td><td>65.6% (9) </td><td>57.5% (11)</td><td>72.6% (5) </td></tr>\\n<tr><td>mean (r)  </td><td>1.27       </td><td>3.00      </td><td>3.47     </td><td>5.07      </td><td>5.13      </td><td>5.73      </td><td>5.80      </td><td>6.33      </td><td>7.00      </td><td>7.60      </td><td>7.80      </td><td>9.27      </td><td>6.40      </td></tr>\\n</tbody>\\n</table>'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "dataset_to_id= {\n",
    "        1: \"fertility\",\n",
    "    2: \"german\",\n",
    "    3: \"diabetes\",\n",
    "    4: \"ionosphere\",\n",
    "    5: \"planning\",\n",
    "    6: \"australian\",\n",
    "    7: \"DWTC\",\n",
    "    8: \"EMNIST\",\n",
    "    9: \"glass\",\n",
    "    10: \"olivetti\",\n",
    "    11: \"CIFAR-10\",\n",
    "    12: \"haberman\",\n",
    "    13: \"zoo\",\n",
    "    14: \"flags\",\n",
    "    15: \"heart\",\n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "dataset_names = ['strategy'] + [dataset_to_id[id] for id in df['dataset_id'].unique()]\n",
    "strategy_results = []\n",
    "strategy_results.append(dataset_names)\n",
    "\n",
    "def get_ranks(l):\n",
    "    uniques = np.unique(l)\n",
    "    argssorted = np.flip(np.argsort(uniques))\n",
    "    mapping = dict(zip(uniques, argssorted))\n",
    "    ranks = [mapping[x] for x in l]\n",
    "    \n",
    "    m2 = {}\n",
    "    last_value = -1\n",
    "    increase = 0\n",
    "    for i, v in enumerate(sorted(ranks)):\n",
    "        if last_value == v:\n",
    "            increase += 1\n",
    "            m2[v] = v\n",
    "        else:\n",
    "            m2[v] = v+increase\n",
    "        last_value = v\n",
    "    \n",
    "    #ranks = [m2[x] for x in ranks]\n",
    "    return ranks\n",
    "\n",
    "for strategy in df['strategy'].unique():\n",
    "    results = [strategy]\n",
    "    for dataset in df['dataset_id'].unique():\n",
    "        results.append(\"{:.40f}\".format(df.loc[(df['dataset_id']==dataset) & (df['strategy']==strategy)]['f1_auc'].mean()))\n",
    "         \n",
    "        results = [r if r != \"nan\" else -0 for r in results] \n",
    "     \n",
    "    strategy_results.append(results)\n",
    "    \n",
    "results_df = pd.DataFrame(strategy_results[1:], columns=strategy_results[0])\n",
    "results_df = results_df.set_index(\"strategy\")\n",
    "results_df = results_df.replace(-1, np.NaN)\n",
    "results_df = results_df.apply(pd.to_numeric)\n",
    "\n",
    "results_df['mean'] = results_df.T.mean(skipna=True)\n",
    "\n",
    "\n",
    "ranking_only_table = []\n",
    "transposed_results = list(map(list, zip(*strategy_results)))\n",
    "new_results = [transposed_results[0]]\n",
    "for row in transposed_results[1:]:\n",
    "    row_sorting = get_ranks(np.array(row[1:]))\n",
    "    row = row[:1] + [\"{:.1%} ({})\".format(float(a),b) for a,b in zip(row[1:], row_sorting)]\n",
    "    new_results.append(row)\n",
    "    ranking_only_table.append(row[:1] + row_sorting)\n",
    "\n",
    "row = [\"mean %\"] + results_df['mean'].to_list()\n",
    "row_sorting = get_ranks(np.array(row[1:]))\n",
    "\n",
    "row = row[:1] + [\"{:.1%} ({})\".format(float(a),b) for a,b in zip(row[1:], row_sorting)]\n",
    "new_results.append(row)\n",
    "\n",
    "\n",
    "ranking_only_table = [transposed_results[0]] + ranking_only_table\n",
    "new_results.append([\"mean (r)\"] + [\"{:1.2f}\".format(sum(r)/len(r)) for r in list(map(list, zip(*ranking_only_table[1:])))[1:]])\n",
    "\n",
    "\n",
    "# sort colmuns manually\n",
    "df2 = pd.DataFrame(new_results[1:], columns=new_results[0])\n",
    "df2 = df2[['strategy', 'NN Single', 'NN Batch', 'MM', 'QBC', 'LC', 'GD', 'BMDR', 'Ent', 'LAL', 'SPAL', 'EER', 'QUIRE','Rand']]\n",
    "\n",
    "order = ['strategy'] + sorted([v for v in dataset_to_id.values()], key=lambda v: v.upper()) + ['mean %', 'mean (r)']\n",
    "lol = [df2.columns.values.tolist()] + df2.values.tolist()\n",
    "lol = sorted(lol, key=lambda l: order.index(l[0]))\n",
    "\n",
    "with open('../paper/figures/ali_f1auc_table.tex2', \"w\") as f:\n",
    "    tex_code = tabulate(lol, headers=\"firstrow\", tablefmt=\"latex_booktabs\")\n",
    "    splitted = [\"\\\\fontseries{b}\\\\selectfont{\" + t[1:-1] + \"}\" if \"(0)\" in t else t for t in tex_code.split(\"&\") ]\n",
    "    \n",
    "    \n",
    "    tex_code = \"&\".join(splitted)\n",
    "    tex_code = tex_code.replace(\"llllllllllllll\", \"L{1.4cm}cccccccccccccc\")\n",
    "    \n",
    "    tex_code = tex_code.replace(\"tabular\", \"tabularx\")\n",
    "    tex_code = tex_code.replace(\"begin{tabularx}\", \"begin{tabularx}{\\linewidth}\")\n",
    "\n",
    "    tex_code = tex_code.replace(\"mean \\%\", \"\\\\midrule \\n mean \\%\")\n",
    "    tex_code = tex_code.replace(\" \\\\% \", \"~~~\")\n",
    "    tex_code = tex_code.replace(\"\\\\% \", \" \")\n",
    "    tex_code = tex_code.replace(\"~~~\", \" \\\\%\")\n",
    "    for ind in range(0,100):\n",
    "        tex_code = tex_code.replace(\" 0.0 (\"+str(ind)+\")\", \"\")\n",
    "        \n",
    "    tex_code = tex_code.replace(\"1.36\", \"\\\\fontseries{b}\\\\selectfont{1.36}\")\n",
    "    tex_code = tex_code.replace(\" \\\\\\\\\", \"\\\\\\\\\")\n",
    "    tex_code = tex_code.replace(\"cc\", \"rr\")\n",
    "    tex_code = tex_code.replace(\"strategy\", \"\")\n",
    "    f.write(tex_code)\n",
    "    \n",
    "tabulate(lol, headers=\"firstrow\", tablefmt=\"html\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ALiPy-8c4hTVCH",
   "language": "python",
   "name": "alipy-8c4htvch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}