Demo_on_SVM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import numpy as np\n",
    "from scipy import sparse\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "from grad_hinge import batch_grad_hinge_loss, grad_hinge_loss_theta, hessian_hingle_loss_theta\n",
    "from grad_hinge import inverse_hvp_hinge_newtonCG\n",
    "\n",
    "from dataset import load_data_v1,select_from_one_class\n",
    "\n",
    "import argparse\n",
    "import time\n",
    "import pdb\n",
    "import os\n",
    "\n",
    "np.random.seed(2019)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select the dataset used\n",
    "# since the processed data set is too large to be submitted in supplementary material,\n",
    "# this notebook cannot be run but only for demonstration.\n",
    "dataset_name = \"cancer\"\n",
    "# parameter for the sigmoid sampling function\n",
    "sigmoid_k = 10\n",
    "# sample ratio\n",
    "sample_ratio = 0.9\n",
    "flip_ratio = 0.4\n",
    "\n",
    "acc_func = lambda x,y: (x==y).sum() / y.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train, nr sample 350, nr feature 11\n",
      "x_va,    nr sample 150, nr feature 11\n",
      "x_te,    nr sample 183, nr feature 11\n",
      "Tr: Pos 164 Neg 186\n",
      "Va: Pos 33 Neg 117\n",
      "Te: Pos 42 Neg 141\n",
      "Load data, cost 0.0 sec\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "# load data, pick 30% as the Va set\n",
    "x_train,y_train,x_va,y_va,x_te,y_te = load_data_v1(dataset_name,va_ratio=0.3)\n",
    "print(\"x_train, nr sample {}, nr feature {}\".format(x_train.shape[0],x_train.shape[1]))\n",
    "print(\"x_va,    nr sample {}, nr feature {}\".format(x_va.shape[0],x_va.shape[1]))\n",
    "print(\"x_te,    nr sample {}, nr feature {}\".format(x_te.shape[0],x_te.shape[1]))\n",
    "print(\"Tr: Pos {} Neg {}\".format(y_train[y_train==1].shape[0],y_train[y_train==0].shape[0]))\n",
    "print(\"Va: Pos {} Neg {}\".format(y_va[y_va==1].shape[0],y_va[y_va==0].shape[0]))\n",
    "print(\"Te: Pos {} Neg {}\".format(y_te[y_te==1].shape[0],y_te[y_te==0].shape[0]))\n",
    "print(\"Load data, cost {:.1f} sec\".format(time.time()-start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=False,\n",
       "          intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "          multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "          verbose=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the subset samples number\n",
    "num_tr_sample = x_train.shape[0]\n",
    "obj_sample_size = int(sample_ratio * num_tr_sample)\n",
    "\n",
    "# define the full-set-model \\hat{\\theta}\n",
    "clf = LinearSVC(loss=\"squared_hinge\", dual=False, fit_intercept=False)\n",
    "\n",
    "# flip labels\n",
    "idxs = np.arange(y_train.shape[0])\n",
    "np.random.shuffle(idxs)\n",
    "num_flip = int(flip_ratio * len(idxs))\n",
    "y_train[idxs[:num_flip]] = np.logical_xor(np.ones(num_flip), y_train[idxs[:num_flip]]).astype(int)\n",
    "                                                                                              \n",
    "# zero to -1\n",
    "y_train[y_train == 0] = -1\n",
    "y_va[y_va == 0] = -1\n",
    "y_te[y_te == 0] = -1\n",
    "\n",
    "\n",
    "clf.fit(x_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[FullSet] Va acc 0.820000\n",
      "[FullSet] Te acc 0.857923\n"
     ]
    }
   ],
   "source": [
    "# on Va\n",
    "y_va_pred = clf.predict(x_va)\n",
    "full_acc = acc_func(y_va, y_va_pred)\n",
    "weight_ar = clf.coef_.flatten()\n",
    "# on Te\n",
    "y_te_pred = clf.predict(x_te)\n",
    "full_te_acc = acc_func(y_te,y_te_pred)\n",
    "full_te_auc = roc_auc_score(y_te,y_te_pred)\n",
    "# print full-set-model results\n",
    "print(\"[FullSet] Va acc {:.6f}\".format(full_acc))\n",
    "print(\"[FullSet] Te acc {:.6f}\".format(full_te_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Function value: -0.08498508673969181\n",
      "Split function value: 0.0, -0.08498508673969181\n",
      "Optimization terminated successfully.\n",
      "         Current function value: -0.084985\n",
      "         Iterations: 1\n",
      "         Function evaluations: 2\n",
      "         Gradient evaluations: 2\n",
      "         Hessian evaluations: 1\n",
      "implicit hessian-vector products mean: -0.07872199256311764\n",
      "implicit hessian-vector products norm: 0.29152201758991003\n",
      "Inverse HVP took 0.0 sec\n",
      "The Influence function's computation completed, cost 0.0 sec\n"
     ]
    }
   ],
   "source": [
    "# get time cost for computing the IF\n",
    "if_start_time = time.time()\n",
    "test_grad_loss_val = grad_hinge_loss_theta(y_va,y_va_pred,x_va)\n",
    "tr_pred = clf.predict(x_train)\n",
    "# tr_pred = clf.decision_function(x_train)\n",
    "batch_size = 10000\n",
    "# computing the inverse Hessian-vector-product\n",
    "iv_hvp = inverse_hvp_hinge_newtonCG(x_train,y_train,tr_pred,test_grad_loss_val,1e-5,True,1.0)\n",
    "# get influence score\n",
    "total_batch = int(np.ceil(x_train.shape[0] / float(batch_size)))\n",
    "predicted_loss_diff = []\n",
    "for idx in range(total_batch):\n",
    "    train_grad_loss_val = batch_grad_hinge_loss(y_train[idx*batch_size:(idx+1)*batch_size],\n",
    "        tr_pred[idx*batch_size:(idx+1)*batch_size],\n",
    "        x_train[idx*batch_size:(idx+1)*batch_size],)\n",
    "\n",
    "    predicted_loss_diff.extend(np.array(train_grad_loss_val.dot(iv_hvp)).flatten())\n",
    "    \n",
    "predicted_loss_diffs = np.asarray(predicted_loss_diff)\n",
    "duration = time.time() - if_start_time\n",
    "print(\"The Influence function's computation completed, cost {:.1f} sec\".format(duration))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================================\n",
      "IF Stats: mean -0.1603376189, max 0.7835683592, min -0.8479409791\n",
      "Pi Stats: [0.00814249 0.14694704 0.5        0.5        0.5       ]\n",
      "============================================================\n",
      "Result Summary on Va\n",
      "[SigUIDS]  acc 0.920000, # 314\n",
      "[Random]   acc 0.720000, # 315\n",
      "[Full]     acc 0.820000, # 350\n",
      "Result Summary on Te\n",
      "[SigUIDS]  acc 0.961749, # 314\n",
      "[Random]   acc 0.759563, # 315\n",
      "[Full]     acc 0.857923, # 350\n",
      "Result Summary on Te\n",
      "[SigUIDS]  auc 0.950101, # 314\n",
      "[Random]   auc 0.743668, # 315\n",
      "[Full]     auc 0.824215, # 350\n",
      "============================================================\n"
     ]
    }
   ],
   "source": [
    "print(\"==\"*30)\n",
    "print(\"IF Stats: mean {:.10f}, max {:.10f}, min {:.10f}\".format(\n",
    "    predicted_loss_diffs.mean(), predicted_loss_diffs.max(), predicted_loss_diffs.min())\n",
    ")\n",
    "# build sampling probability\n",
    "phi_ar = - predicted_loss_diffs\n",
    "IF_interval = phi_ar.max() - phi_ar.min()\n",
    "a_param = sigmoid_k / IF_interval\n",
    "prob_pi = 1 / (1 + np.exp(a_param * phi_ar))\n",
    "print(\"Pi Stats:\",np.percentile(prob_pi,[10,25,50,75,90]))\n",
    "\n",
    "# Do subsampling\n",
    "pos_idx = select_from_one_class(y_train,prob_pi,1,sample_ratio)\n",
    "neg_idx = select_from_one_class(y_train,prob_pi,-1,sample_ratio)\n",
    "sb_idx = np.union1d(pos_idx,neg_idx)\n",
    "sb_x_train = x_train[sb_idx]\n",
    "sb_y_train = y_train[sb_idx]\n",
    "\n",
    "# Train the subset-model \\tilde{\\theta}\n",
    "clf.fit(sb_x_train,sb_y_train)\n",
    "y_va_pred = clf.predict(x_va)\n",
    "sb_acc = (y_va == y_va_pred).sum() / y_va.shape[0]\n",
    "sb_weight = clf.coef_.flatten()\n",
    "diff_w_norm = np.linalg.norm(weight_ar - sb_weight)\n",
    "sb_size = sb_x_train.shape[0]\n",
    "y_te_pred = clf.predict(x_te)\n",
    "sb_te_auc = roc_auc_score(y_te, y_te_pred)\n",
    "y_te_pred = clf.predict(x_te)\n",
    "sb_te_acc = (y_te == y_te_pred).sum() / y_te.shape[0]\n",
    "\n",
    "\n",
    "# baseline: random sampling\n",
    "u_idxs = np.arange(x_train.shape[0])\n",
    "uniform_idxs = np.random.choice(u_idxs,obj_sample_size,replace=False)\n",
    "us_x_train = x_train[uniform_idxs]\n",
    "us_y_train = y_train[uniform_idxs]\n",
    "clf.fit(us_x_train, us_y_train)\n",
    "y_va_pred = clf.predict(x_va)\n",
    "us_acc = (y_va == y_va_pred).sum() / y_va.shape[0]\n",
    "us_size = us_x_train.shape[0]\n",
    "y_te_pred = clf.predict(x_te)\n",
    "us_te_auc = roc_auc_score(y_te, y_te_pred)\n",
    "y_te_pred = clf.predict(x_te)\n",
    "us_te_acc = (y_te == y_te_pred).sum() / y_te.shape[0]\n",
    "\n",
    "\n",
    "\n",
    "print(\"==\"*30)\n",
    "print(\"Result Summary on Va\")\n",
    "print(\"[SigUIDS]  acc {:.6f}, # {}\".format(sb_acc,sb_size))\n",
    "print(\"[Random]   acc {:.6f}, # {}\".format(us_acc,us_size))\n",
    "print(\"[Full]     acc {:.6f}, # {}\".format(full_acc,num_tr_sample))\n",
    "print(\"Result Summary on Te\")\n",
    "print(\"[SigUIDS]  acc {:.6f}, # {}\".format(sb_te_acc,sb_size))\n",
    "print(\"[Random]   acc {:.6f}, # {}\".format(us_te_acc,us_size))\n",
    "print(\"[Full]     acc {:.6f}, # {}\".format(full_te_acc,num_tr_sample))\n",
    "print(\"Result Summary on Te\")\n",
    "print(\"[SigUIDS]  auc {:.6f}, # {}\".format(sb_te_auc,sb_size))\n",
    "print(\"[Random]   auc {:.6f}, # {}\".format(us_te_auc,us_size))\n",
    "print(\"[Full]     auc {:.6f}, # {}\".format(full_te_auc,num_tr_sample))\n",
    "print(\"==\"*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}