AL_CV_parameter_tuning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lukekolbe/AL-in-CreditScoring/blob/main/AL_CV_parameter_tuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bs2HdyuEA3Pu",
        "outputId": "670f7bf4-eab2-4814-ffb1-6fba5c790312"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive', force_remount=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n-r6TwUlA8H5",
        "outputId": "eef35c56-5fd6-430e-d6f2-65fbca8adc48"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found existing installation: cvxpy 1.0.31\n",
            "Uninstalling cvxpy-1.0.31:\n",
            "  Successfully uninstalled cvxpy-1.0.31\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting cvxpy\n",
            "  Downloading cvxpy-1.2.1-cp37-cp37m-manylinux_2_24_x86_64.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 28.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: ecos>=2 in /usr/local/lib/python3.7/dist-packages (from cvxpy) (2.0.10)\n",
            "Requirement already satisfied: scs>=1.1.6 in /usr/local/lib/python3.7/dist-packages (from cvxpy) (3.2.0)\n",
            "Requirement already satisfied: osqp>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from cvxpy) (0.6.2.post0)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from cvxpy) (1.7.3)\n",
            "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from cvxpy) (1.21.6)\n",
            "Requirement already satisfied: qdldl in /usr/local/lib/python3.7/dist-packages (from osqp>=0.4.1->cvxpy) (0.1.5.post2)\n",
            "Installing collected packages: cvxpy\n",
            "Successfully installed cvxpy-1.2.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/NUAA-AL/alipy.git\n",
            "  Cloning https://github.com/NUAA-AL/alipy.git to /tmp/pip-req-build-rtzcxcs3\n",
            "  Running command git clone -q https://github.com/NUAA-AL/alipy.git /tmp/pip-req-build-rtzcxcs3\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from alipy==1.2.5) (1.21.6)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from alipy==1.2.5) (1.7.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from alipy==1.2.5) (1.0.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from alipy==1.2.5) (3.2.2)\n",
            "Requirement already satisfied: prettytable in /usr/local/lib/python3.7/dist-packages (from alipy==1.2.5) (3.3.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->alipy==1.2.5) (1.4.4)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->alipy==1.2.5) (3.0.9)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->alipy==1.2.5) (2.8.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->alipy==1.2.5) (0.11.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->alipy==1.2.5) (4.1.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->alipy==1.2.5) (1.15.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prettytable->alipy==1.2.5) (0.2.5)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from prettytable->alipy==1.2.5) (4.12.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->prettytable->alipy==1.2.5) (3.8.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->alipy==1.2.5) (3.1.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->alipy==1.2.5) (1.1.0)\n",
            "Building wheels for collected packages: alipy\n",
            "  Building wheel for alipy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for alipy: filename=alipy-1.2.5-py3-none-any.whl size=121053 sha256=80b8626114d3580799012597aa7d4f746b6cee2dd3a3c6d6f35f2275555b4cff\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-h51k_m8d/wheels/e6/2f/a4/845ce9812f1a5c537c1bda39a26e36234a5a59535269feac13\n",
            "Successfully built alipy\n",
            "Installing collected packages: alipy\n",
            "Successfully installed alipy-1.2.5\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: imbalanced-learn in /usr/local/lib/python3.7/dist-packages (0.8.1)\n",
            "Collecting imbalanced-learn\n",
            "  Downloading imbalanced_learn-0.9.1-py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 21.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from imbalanced-learn) (1.21.6)\n",
            "Requirement already satisfied: joblib>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from imbalanced-learn) (1.1.0)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.7/dist-packages (from imbalanced-learn) (1.7.3)\n",
            "  Downloading imbalanced_learn-0.9.0-py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 47.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from imbalanced-learn) (1.0.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from imbalanced-learn) (3.1.0)\n",
            "Installing collected packages: imbalanced-learn\n",
            "  Attempting uninstall: imbalanced-learn\n",
            "    Found existing installation: imbalanced-learn 0.8.1\n",
            "    Uninstalling imbalanced-learn-0.8.1:\n",
            "      Successfully uninstalled imbalanced-learn-0.8.1\n",
            "Successfully installed imbalanced-learn-0.9.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting scikit-optimize\n",
            "  Downloading scikit_optimize-0.9.0-py2.py3-none-any.whl (100 kB)\n",
            "\u001b[K     |████████████████████████████████| 100 kB 8.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.0.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.7.3)\n",
            "Collecting pyaml>=16.9\n",
            "  Downloading pyaml-21.10.1-py2.py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.21.6)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from pyaml>=16.9->scikit-optimize) (3.13)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.20.0->scikit-optimize) (3.1.0)\n",
            "Installing collected packages: pyaml, scikit-optimize\n",
            "Successfully installed pyaml-21.10.1 scikit-optimize-0.9.0\n"
          ]
        }
      ],
      "source": [
        "### FOR RUNNING ON COLAB:\n",
        "\n",
        "!pip uninstall cvxpy -y\n",
        "!pip install cvxpy\n",
        "!pip install git+https://github.com/NUAA-AL/alipy.git\n",
        "!pip install -U imbalanced-learn\n",
        "!pip install scikit-optimize"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip freeze"
      ],
      "metadata": {
        "id": "3osHbuDSX5QN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VsS3enpvA9vz"
      },
      "outputs": [],
      "source": [
        "############ LIBRARIES\n",
        "\n",
        "import os\n",
        "import time\n",
        "import datetime\n",
        "import random\n",
        "import multiprocessing\n",
        "import pickle\n",
        "import re\n",
        "import copy\n",
        "import gc\n",
        "import sys\n",
        "import json\n",
        "gc.enable()\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "pd.set_option('display.max_columns', None)\n",
        "\n",
        "from itertools import cycle\n",
        "\n",
        "\n",
        "import scipy.stats\n",
        "\n",
        "#from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "#from sklearn.model_selection import StratifiedKFold  ##### what is this used for?\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.metrics import brier_score_loss\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import fbeta_score\n",
        "\n",
        "# FOR CROSS VALIDATED HYPERPARAMETER TUNING\n",
        "# use imblearn pipeline instead of sklearn pipeline to skip AL sampling process in the prediction phase\n",
        "from imblearn.pipeline import Pipeline\n",
        "from imblearn import FunctionSampler\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "\n",
        "\n",
        "from sklearn.ensemble import RandomForestClassifier \n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "from alipy import ToolBox\n",
        "from alipy import query_strategy\n",
        "from alipy.index import IndexCollection\n",
        "from alipy import data_manipulate\n",
        "import cvxpy\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lt5v7qnlA_3_"
      },
      "outputs": [],
      "source": [
        "os.chdir('/gdrive/My Drive/ACTIVE LEARNING THESIS/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "ouyUQiiGgQ_y",
        "outputId": "24fb56c4-367e-45c5-d21c-704a369c7d36"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/gdrive/My Drive/ACTIVE LEARNING THESIS'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "os.getcwd()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cal7cW8gBDw5"
      },
      "source": [
        "# CV helper functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X9kg8erLH2Iy"
      },
      "outputs": [],
      "source": [
        "############ RANDOMNESS\n",
        "# seed function\n",
        "def seed_everything(seed):\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    \n",
        "# set seed\n",
        "seed = 30\n",
        "seed_everything(seed)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uQYLaSjfBLdR"
      },
      "outputs": [],
      "source": [
        "# get strategy\n",
        "\n",
        "def strategy_getter(X, y, strategy_name=\"QueryInstanceRandom\", train_idx = None, **kwargs):\n",
        "    \"\"\"Return the query strategy object from alipy package\"\"\"\n",
        "    \n",
        "    try:\n",
        "        exec(\"from alipy.query_strategy import \" + strategy_name)\n",
        "    except:\n",
        "        raise KeyError(\"Strategy \" + strategy_name + \" is not implemented in ALiPy.\")\n",
        "    strategy = None\n",
        "    \n",
        "    if train_idx is not None:\n",
        "      strategy = eval(strategy_name + \"(X=X, y=y, train_idx = train_idx, **kwargs)\")\n",
        "    else:\n",
        "      strategy = eval(strategy_name + \"(X=X, y=y, **kwargs)\")\n",
        "          \n",
        "    # print(strategy)\n",
        "    return strategy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rlO4CCnKWF4y"
      },
      "outputs": [],
      "source": [
        "#Custom Data Scaler for use in tuning pipeline\n",
        "class CustomScaler(BaseEstimator, TransformerMixin):\n",
        "  \n",
        "  def __init__(self, with_centering=True, with_scaling=True, seed=30):\n",
        "        self.seed = seed\n",
        "        self.with_centering = with_centering\n",
        "        self.with_scaling = with_scaling\n",
        "        self.scaler = RobustScaler(with_centering=self.with_centering, with_scaling = self.with_scaling)\n",
        "\n",
        "\n",
        "  #estimator method\n",
        "  def fit(self, X, y):\n",
        "    sss = StratifiedShuffleSplit(n_splits=1, test_size=int(0.5*len(X)), random_state=self.seed) \n",
        "    for label, unlabel in sss.split(X=X, y=y):\n",
        "      label_idx, unlabel_idx = np.asarray(label), np.asarray(unlabel)    \n",
        "\n",
        "      self.scaler.fit(X[label_idx,:])\n",
        "      return self\n",
        "\n",
        "  #transformation\n",
        "  def transform(self, X, y = None):\n",
        "    self.X = self.scaler.transform(X)\n",
        "    return self.X\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iFRxIj1rBNFI"
      },
      "outputs": [],
      "source": [
        "# Custom Sampler Class, only run in the fit phase of the pipeline, skipped in predict phase\n",
        "#####################################\n",
        "def AL_resampler(X=None\n",
        "              , y=None\n",
        "              , strategy_name = None\n",
        "              , pass_index=False\n",
        "              , model = None\n",
        "              , seed=30\n",
        "              , **kwargs):\n",
        "\n",
        "  \n",
        "  os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "  random.seed(seed)\n",
        "  np.random.seed(seed)\n",
        "  \n",
        "  # re-create the sample that was used to scale the data!\n",
        "  # this is done so that the scaling is only done with data that is labeled, so that there is no leakage\n",
        "  # this is the intial labeled sample that all strategies share\n",
        "  sss = StratifiedShuffleSplit(n_splits=1, test_size=int(0.5*len(X)), random_state=seed) # THESE ARE POSITIONS, NOT INDICES    \n",
        "  for label, unlabel in sss.split(X=X, y=y):\n",
        "    label_idx, unlabel_idx = np.asarray(label), np.asarray(unlabel)    \n",
        "        \n",
        "\n",
        "  n_instances = int(0.5*len(unlabel_idx))\n",
        "\n",
        "  ##############\n",
        "  # initialize AL strategy\n",
        "  ##############\n",
        "  t_idx = np.concatenate((label_idx, unlabel_idx))\n",
        "\n",
        "  if pass_index and not strategy_name == 'QueryInstanceLAL':\n",
        "    strategy = strategy_getter(X, y, strategy_name, t_idx, **kwargs)\n",
        "\n",
        "  elif not pass_index and not strategy_name == 'QueryInstanceLAL':\n",
        "    strategy = strategy_getter(X, y, strategy_name, **kwargs)\n",
        "\n",
        "  # special case because of extra RF; might be prohibitively expensive to tune\n",
        "  elif strategy_name == 'QueryInstanceLAL':\n",
        "\n",
        "    param_dict = {**kwargs}\n",
        "    reg_est=param_dict.pop('reg_est', None)\n",
        "    reg_depth=param_dict.pop('reg_depth', None)\n",
        "    reg_feat=param_dict.pop('reg_feat', None)\n",
        "    if reg_feat > np.shape(X)[1]:\n",
        "      reg_feat = np.shape(X)[1]\n",
        "\n",
        "    strategy = strategy_getter(X, y, strategy_name = strategy_name, mode='LAL_iterative', train_slt=False, **param_dict)\n",
        "    starttime = time.time()\n",
        "    strategy.download_data()\n",
        "    print('duration of data download', starttime - time.time())\n",
        "    strategy.train_selector_from_file(reg_est=reg_est, reg_depth=reg_depth, feat=reg_feat)\n",
        "\n",
        "  ##############\n",
        "  # run AL selection once\n",
        "  ##############\n",
        "\n",
        "  #fit model then pass it as argument to some AL strategies\n",
        "  if not model == None:\n",
        "    model.fit(X[label_idx], y[label_idx])\n",
        "\n",
        "  if strategy_name == 'QueryInstanceQUIRE': #strategy quire has no batch mode\n",
        "    label_idx = IndexCollection(label_idx)\n",
        "    unlabel_idx = IndexCollection(unlabel_idx)\n",
        "\n",
        "    print('starting selection')\n",
        "    for n in range(n_instances):\n",
        "      selection = strategy.select(label_index=label_idx.index, unlabel_index=unlabel_idx.index)[0] # returns a list of len one, hence the [0]\n",
        "      label_idx.update(selection)\n",
        "      unlabel_idx.difference_update(selection)\n",
        "      if n%10 == 0:\n",
        "        print(f\"selected {n} cases\")\n",
        "    label_idx = label_idx.index\n",
        "\n",
        "  elif strategy_name in ['QueryInstanceBMDR', 'QueryInstanceSPAL']:\n",
        "    select_idx = strategy.select(label_idx, unlabel_idx, model=model,batch_size=n_instances, qp_solver = 'OSQP')\n",
        "    label_idx = np.concatenate((label_idx, select_idx))\n",
        "\n",
        "  elif model == None:\n",
        "    select_idx = strategy.select(IndexCollection(label_idx), IndexCollection(unlabel_idx), batch_size=n_instances)\n",
        "    label_idx = np.concatenate((label_idx, select_idx))\n",
        "  \n",
        "  elif not model ==None:\n",
        "    select_idx = strategy.select(label_idx, unlabel_idx, model=model, batch_size=n_instances)\n",
        "    label_idx = np.concatenate((label_idx, select_idx))\n",
        "\n",
        "\n",
        "  return X[label_idx,:], y[label_idx]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GAxITIsasgrx"
      },
      "outputs": [],
      "source": [
        "def append_record(record, filename):\n",
        "    with open(f'metaparameters/{filename}', 'a') as f:\n",
        "        json.dump(record, f)\n",
        "        f.write(os.linesep)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O3TKLv6JBTln"
      },
      "source": [
        "# Load Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U4b6VBKDBFRF",
        "outputId": "7b640a77-520f-4230-bc6b-7dd25f5956b4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "X type:  <class 'numpy.ndarray'> X shape:  (690, 42) y shape:  (690,) y mean:  0.4449275362318841\n",
            "140421149539664 140421705635216\n"
          ]
        }
      ],
      "source": [
        "############ DATA IMPORT\n",
        "\n",
        "## available datasets\n",
        "\n",
        "  # OK gmsc          # shape:  (150000, 68)\n",
        "  # OK uk            # shape:  (30000, 51), y mean:  0.04\n",
        "  # OK lendingclub   # shape:  (41623, 114) y mean:  0.1331235134420873    \n",
        "  # OK bene2         # shape:  (7190, 28)\n",
        "  # bene1            # shape:  (3123, 18)\n",
        "  # hmeq             # shape:  (5960, 20)\n",
        "  # australian       # shape:  (690, 42)\n",
        "  # german           # shape:  (1000, 61)\n",
        "  # thomas           # shape:  (1225, 28)\n",
        "  # pakdd            # shape:  (50000, 373), y mean:  0.26082\n",
        "\n",
        "dataset = \"australian\"\n",
        "#df = pd.read_csv('//home//RDC//kolbeluk1//AL_THESIS//prepared_data//{}.csv'.format(dataset)) #linux path\n",
        "# C:\\\\Users\\\\kolbeluk1\\\\AL_THESIS\n",
        "#df = pd.read_csv('C:\\\\Users\\\\kolbeluk1\\\\AL_THESIS\\\\prepared_data\\\\{}.csv'.format(dataset))\n",
        "\n",
        "df = pd.read_csv('/gdrive/My Drive/ACTIVE LEARNING THESIS/prepared_data/{}.csv'.format(dataset))\n",
        "\n",
        "# remove NA\n",
        "df = df.dropna()\n",
        "df.reset_index(drop = True, inplace = True)\n",
        "\n",
        "# extract label\n",
        "df['BAD'][df['BAD']=='BAD']  = 1\n",
        "df['BAD'][df['BAD']=='GOOD'] = 0\n",
        "df['BAD'] = df['BAD'].astype('int')\n",
        "\n",
        "\n",
        "y_temp = df['BAD']\n",
        "del df['BAD']\n",
        "\n",
        "#one hot encoding\n",
        "df = pd.get_dummies(df)\n",
        "\n",
        "#transform to numpy array >> same location for df and X\n",
        "X = df.to_numpy()\n",
        "y = y_temp.to_numpy()\n",
        "\n",
        "print(\"X type: \", type(X), \"X shape: \",X.shape,\"y shape: \", y.shape, \"y mean: \", np.mean(y))\n",
        "print (id(X), id(df))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "9QeXHZEBH6-k",
        "outputId": "f3fa440d-789a-4b63-bb5b-c336df963f0c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'australian_tuned-params'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "# append_record: helper function that adds best-parameter for every model to dict and saves it\n",
        "# careful when re-tuning: formula just adds entries, does not delete existing. Multiple specs for single model are possible. Best run once without pre-existing parameter file for this dataset.\n",
        "filename = f'{dataset}_tuned-params'\n",
        "filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NqiO1UexBVTr"
      },
      "source": [
        "# Split off Validation Set\n",
        "\n",
        "implement scheme for splitting a validation set"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lbD-fnJ5Bc5w",
        "outputId": "d1b23bcc-e7e9-42ff-faa0-883a5eec576d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(690, 42) (690, 42) folds:  10\n"
          ]
        }
      ],
      "source": [
        "# determine number of folds based on dataset size, reduce size of training if necessary (stratified split)\n",
        "if len(y) > 25000:\n",
        "    validation_size = 7500\n",
        "    folds = 3\n",
        "    sss = StratifiedShuffleSplit(n_splits=1, test_size=validation_size, random_state=seed)\n",
        "    for model, validation in sss.split(X=X, y=y):\n",
        "        model_idx, validation_idx = np.asarray(model), np.asarray(validation)    \n",
        "\n",
        "    print(np.sum(model_idx), np.sum(validation_idx))\n",
        "    X_val, y_val = X[validation_idx,:], y[validation_idx]\n",
        "\n",
        "elif len(y) < 2000:\n",
        "    folds = 10\n",
        "    X_val, y_val = X, y\n",
        "\n",
        "else:\n",
        "    folds = 5\n",
        "    X_val, y_val = X, y\n",
        "    \n",
        "print(np.shape(X), np.shape(X_val), 'folds: ', folds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WkNXYuN1CPs7"
      },
      "source": [
        "# CV Run"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gV0fQCgxCRIZ"
      },
      "outputs": [],
      "source": [
        "# use imblearn pipeline to skip AL sampling process in the prediction phase\n",
        "CLF_pipe = Pipeline(steps=[('scaler', CustomScaler(seed=seed)),\n",
        "                           ('AL_sampler', FunctionSampler(func=AL_resampler)),    # this will trigger a call to __init__\n",
        "                           ('clf', LogisticRegression(random_state=seed))])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EwWdWAqXUJn_",
        "outputId": "d586dcb7-0cfe-4d54-888a-b134027211bf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 10 folds for each of 144 candidates, totalling 1440 fits\n"
          ]
        }
      ],
      "source": [
        "## RUN TUNING FOR CLASSIFIER FIRST!\n",
        "\n",
        "for key in ['random']:\n",
        "\n",
        "  param_grid={'clf__solver': ['liblinear'], \n",
        "              'clf__penalty': ['l1', 'l2'], \n",
        "              'clf__C': [0.001,0.01,0.1,1.,10.,100.], \n",
        "              'clf__max_iter': [50, 100, 250, 500], \n",
        "              'clf__tol': [0.001, 0.0001, 0.00001],               \n",
        "              'AL_sampler__kw_args': [{'strategy_name': 'QueryInstanceRandom'}]}\n",
        "\n",
        "  grid_search_clf = RandomizedSearchCV(CLF_pipe, param_distributions=param_grid, n_iter = 500, cv=10, n_jobs=20, verbose=5, error_score=\"raise\", refit='roc_auc', scoring=['roc_auc'], random_state=seed)\n",
        "  #random search covers all possible parameters, so it is equivalent to running gridsearch\n",
        "  grid_search_clf.fit(X_val, y_val)\n",
        "\n",
        "  ### SAVE RESULTS\n",
        "  AL_params = copy.deepcopy(grid_search_clf.best_params_['AL_sampler__kw_args'])   #[i] for i in grid_search_clf.best_params_['sampler__kw_args'] if i not in ['key', 'name', 'sample_size']]\n",
        "  AL_params.pop('pass_index', None)\n",
        "  AL_params.pop('model', None)\n",
        "  AL_params.pop('skip', None)\n",
        "\n",
        "  CLF_params = copy.deepcopy(grid_search_clf.best_params_)\n",
        "  CLF_params.pop('AL_sampler__kw_args', None)\n",
        "  CLF_params.pop('outlier__kw_args', None)\n",
        "\n",
        "  for clf_key in list(CLF_params.keys()):\n",
        "    new_key = re.sub(r'clf__', '', clf_key)\n",
        "    CLF_params[new_key] = CLF_params.pop(clf_key)\n",
        "\n",
        "  cv_param_dict = {f'{key}': {#'outlier_rf': outlier_params,\n",
        "                 'AL': AL_params,\n",
        "                 'CLF': CLF_params}}\n",
        "  \n",
        "  append_record(cv_param_dict, filename)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IEWUJKS-Ulvb",
        "outputId": "0696ec1d-efab-4754-8991-8eb53d3c222d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LogisticRegression(max_iter=50, penalty='l1', random_state=30,\n",
            "                   solver='liblinear', tol=1e-05)\n"
          ]
        }
      ],
      "source": [
        "#Define tunable parameters for AL models\n",
        "\n",
        "clf = LogisticRegression(random_state=seed)\n",
        "\n",
        "clf_tuned_params = copy.deepcopy(grid_search_clf.best_params_)\n",
        "clf_tuned_params.pop('AL_sampler__kw_args', None)\n",
        "\n",
        "for clf_key in list(clf_tuned_params.keys()):\n",
        "  new_key = re.sub(r'clf__', '', clf_key)\n",
        "  clf_tuned_params[new_key] = clf_tuned_params.pop(clf_key)\n",
        "\n",
        "clf.set_params(**clf_tuned_params)\n",
        "print(str(clf))\n",
        "\n",
        "AL_pipe = Pipeline(steps=[#('outlier', FunctionSampler(func=outlier_rejection)),\n",
        "                      ('scaler', CustomScaler(seed=seed)),\n",
        "                      ('AL_sampler', FunctionSampler(func=AL_resampler)),    # this will trigger a call to __init__\n",
        "                      ('clf', clf)])\n",
        "\n",
        "cv_parameters = {'unc': [{'strategy_name': 'QueryInstanceUncertainty', 'model': clf, 'measure': m} for m in ['entropy', 'least_confident', 'margin', 'distance_to_boundary']]\n",
        "                ,'qbc': [{'strategy_name': 'QueryInstanceQBC', 'model': clf, 'method': 'query_by_bagging', 'disagreement': d} for d in ['vote_entropy', 'KL_divergence']]\n",
        "                ,'dw': [{'strategy_name': 'QueryInstanceDensityWeighted', 'model': clf, 'uncertainty_meansure': u , 'distance': d, 'beta': b} for u in ['least_confident', 'margin', 'entropy'] for d in ['cityblock', 'cosine', 'euclidean'] for b in [0.5, 1, 2]]\n",
        "                ,'density':[{'strategy_name': 'QueryInstanceGraphDensity', 'pass_index': True, 'metric': m} for m in ['canberra', 'jaccard', 'cosine', 'hamming']]\n",
        "                ,'cors' :  [{'strategy_name': 'QueryInstanceCoresetGreedy', 'pass_index': True, 'distance': d} for d in ['cityblock', 'cosine', 'euclidean']]\n",
        "                ,'lal': [{'strategy_name': 'QueryInstanceLAL', 'cls_est': cls_est, 'reg_est': reg_est, 'reg_depth': reg_depth, 'reg_feat': reg_feat} for cls_est in [16,32,64] for reg_est in [32,64,128] for reg_depth in [5,10,20] for reg_feat in [5,6,7]]\n",
        "                ,'spal': [{'strategy_name': 'QueryInstanceSPAL', 'kernel': 'rbf', 'mu': mu, 'gamma': g, 'lambda_init': li, 'lambda_pace': lp, 'rho':10} for mu in [0.01, 0.1, 1] for g in [0.01, 0.1, 1] for li in [0.01, 0.1, 1] for lp in [0.001, 0.01, 0.1]] #parameter rho is not tuned because it has massive effect on computation times\n",
        "                ,'bmdr': [{'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': b, 'gamma': g, 'rho':10} for b in [100, 1000, 10000] for g in [0.01, 0.1, 1]] # issues with the solver ECOS; parameter rho is not tuned because it has massive effect on computation times\n",
        "                #,'eer' has no tunable parameters\n",
        "                #,'quire' does not run stable enough for tuning on all datasets, some parameter combinations seem to cause issues\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5s7VjzPHgRAE",
        "outputId": "fbc95bb9-0508-4a0c-e5be-8666d0acaa29"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1658736200.6394048\n",
            "Fitting 10 folds for each of 4 candidates, totalling 40 fits\n",
            "unc time: 0.0002417404121822781\n",
            "Fitting 10 folds for each of 2 candidates, totalling 20 fits\n",
            "qbc time: 0.000354471140437656\n",
            "Fitting 10 folds for each of 27 candidates, totalling 270 fits\n",
            "dw time: 0.0016161796119478014\n",
            "Fitting 10 folds for each of 4 candidates, totalling 40 fits\n",
            "density time: 0.03715307745668623\n",
            "Fitting 10 folds for each of 3 candidates, totalling 30 fits\n",
            "cors time: 0.0007801794343524509\n",
            "Fitting 10 folds for each of 81 candidates, totalling 810 fits\n"
          ]
        }
      ],
      "source": [
        "# tune AL models\n",
        "\n",
        "start = time.time()\n",
        "print(start)\n",
        "\n",
        "for key in cv_parameters.keys(): #eer, quire, lal, spal, bmdr\n",
        "#for key in ['bmdr']:\n",
        "  loop_start = time.time()\n",
        "  param_grid={'AL_sampler__kw_args': cv_parameters[key]} \n",
        "  \n",
        "  grid_search_AL = RandomizedSearchCV(AL_pipe, param_distributions=param_grid, n_iter=150, cv=folds, n_jobs=20, verbose=5, error_score=\"raise\", refit='roc_auc', scoring=['roc_auc'], random_state=seed)\n",
        "  grid_search_AL.fit(X_val, y_val)\n",
        "\n",
        "  AL_params = copy.deepcopy(grid_search_AL.best_params_['AL_sampler__kw_args'])  \n",
        "  AL_params.pop('pass_index', None)\n",
        "  AL_params.pop('model', None)\n",
        "  AL_params.pop('skip', None)\n",
        "\n",
        "  CLF_params = copy.deepcopy(grid_search_AL.best_params_)\n",
        "  CLF_params.pop('AL_sampler__kw_args', None)\n",
        "\n",
        "  for clf_key in list(CLF_params.keys()):\n",
        "    new_key = re.sub(r'clf__', '', clf_key)\n",
        "    CLF_params[new_key] = CLF_params.pop(clf_key)\n",
        "\n",
        "  cv_param_dict = {f'{key}': {#'outlier_rf': outlier_params,\n",
        "                 'AL': AL_params,\n",
        "                 'CLF': CLF_params}}\n",
        "  \n",
        "  append_record(cv_param_dict, filename)\n",
        "  \n",
        "  print(f'{key} time:',(time.time() - loop_start)/3600)\n",
        "print('total time:',(time.time() - start)/3600)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mnetRdFaH-pj"
      },
      "outputs": [],
      "source": [
        "#### PRINT RESULTS\n",
        "print(\"Best parameters set found on development set:\", \"\\n\")\n",
        "print(grid_search_AL.best_params_)\n",
        "print(\"Grid scores on development set:\", \"\\n\")\n",
        "means_roc = grid_search_AL.cv_results_['mean_test_roc_auc']\n",
        "stds_roc = grid_search_AL.cv_results_['std_test_roc_auc']\n",
        "for means_roc, stds_roc, params in zip(means_roc, stds_roc, grid_search_AL.cv_results_['params']):\n",
        "  print(\"%0.3f (+/-%0.03f) for %r\"\n",
        "        % (means_roc, stds_roc * 2, params))\n",
        "\n",
        "print(\"\\n\", \"Detailed classification report:\", \"\\n\")\n",
        "print(\"The model is trained on the full development set.\", \"\\n\")\n",
        "print(\"The scores are computed on the full evaluation set.\", \"\\n\")\n",
        "y_pred = grid_search_AL.predict(X_model)\n",
        "print(classification_report(y_model, y_pred))\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "AL_CV_parameter-tuning.ipynb",
      "toc_visible": true,
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}