regression_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_tabnet.tab_model import TabNetRegressor\n",
    "\n",
    "import torch\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "\n",
    "\n",
    "import os\n",
    "import wget\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download census-income dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\"\n",
    "dataset_name = 'census-income'\n",
    "out = Path(os.getcwd()+'/data/'+dataset_name+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File already exists.\n"
     ]
    }
   ],
   "source": [
    "out.parent.mkdir(parents=True, exist_ok=True)\n",
    "if out.exists():\n",
    "    print(\"File already exists.\")\n",
    "else:\n",
    "    print(\"Downloading file...\")\n",
    "    wget.download(url, out.as_posix())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data and split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(out)\n",
    "target = ' <=50K'\n",
    "if \"Set\" not in train.columns:\n",
    "    train[\"Set\"] = np.random.choice([\"train\", \"valid\", \"test\"], p =[.8, .1, .1], size=(train.shape[0],))\n",
    "\n",
    "train_indices = train[train.Set==\"train\"].index\n",
    "valid_indices = train[train.Set==\"valid\"].index\n",
    "test_indices = train[train.Set==\"test\"].index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple preprocessing\n",
    "\n",
    "Label encode categorical features and fill empty cells."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " State-gov 9\n",
      " Bachelors 16\n",
      " Never-married 7\n",
      " Adm-clerical 15\n",
      " Not-in-family 6\n",
      " White 5\n",
      " Male 2\n",
      " United-States 42\n",
      " <=50K 2\n",
      "Set 3\n"
     ]
    }
   ],
   "source": [
    "categorical_columns = []\n",
    "categorical_dims =  {}\n",
    "for col in train.columns[train.dtypes == object]:\n",
    "    print(col, train[col].nunique())\n",
    "    l_enc = LabelEncoder()\n",
    "    train[col] = train[col].fillna(\"VV_likely\")\n",
    "    train[col] = l_enc.fit_transform(train[col].values)\n",
    "    categorical_columns.append(col)\n",
    "    categorical_dims[col] = len(l_enc.classes_)\n",
    "\n",
    "for col in train.columns[train.dtypes == 'float64']:\n",
    "    train.fillna(train.loc[train_indices, col].mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define categorical features for categorical embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "unused_feat = ['Set']\n",
    "\n",
    "features = [ col for col in train.columns if col not in unused_feat+[target]] \n",
    "\n",
    "cat_idxs = [ i for i, f in enumerate(features) if f in categorical_columns]\n",
    "\n",
    "cat_dims = [ categorical_dims[f] for i, f in enumerate(features) if f in categorical_columns]\n",
    "\n",
    "# define your embedding sizes : here just a random choice\n",
    "cat_emb_dim = [5, 4, 3, 6, 2, 2, 1, 10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/work/pytorch_tabnet/abstract_model.py:75: UserWarning: Device used : cuda\n",
      "  warnings.warn(f\"Device used : {self.device}\")\n"
     ]
    }
   ],
   "source": [
    "clf = TabNetRegressor(cat_dims=cat_dims, cat_emb_dim=cat_emb_dim, cat_idxs=cat_idxs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train[features].values[train_indices]\n",
    "y_train = train[target].values[train_indices].reshape(-1, 1)\n",
    "\n",
    "X_valid = train[features].values[valid_indices]\n",
    "y_valid = train[target].values[valid_indices].reshape(-1, 1)\n",
    "\n",
    "X_test = train[features].values[test_indices]\n",
    "y_test = train[target].values[test_indices].reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_epochs = 100 if not os.getenv(\"CI\", False) else 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_tabnet.augmentations import RegressionSMOTE\n",
    "aug = RegressionSMOTE(p=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0  | loss: 0.16632 | train_rmsle: 0.08172 | train_mae: 0.32434 | train_rmse: 0.42037 | train_mse: 0.17671 | valid_rmsle: 0.08229 | valid_mae: 0.32509 | valid_rmse: 0.41638 | valid_mse: 0.17337 |  0:00:01s\n",
      "epoch 1  | loss: 0.13043 | train_rmsle: 0.08851 | train_mae: 0.34694 | train_rmse: 0.43319 | train_mse: 0.18765 | valid_rmsle: 0.08877 | valid_mae: 0.34742 | valid_rmse: 0.43215 | valid_mse: 0.18676 |  0:00:02s\n",
      "epoch 2  | loss: 0.12421 | train_rmsle: 0.07367 | train_mae: 0.31208 | train_rmse: 0.38054 | train_mse: 0.14481 | valid_rmsle: 0.07219 | valid_mae: 0.30876 | valid_rmse: 0.37698 | valid_mse: 0.14211 |  0:00:04s\n",
      "epoch 3  | loss: 0.11857 | train_rmsle: 0.08045 | train_mae: 0.30774 | train_rmse: 0.38188 | train_mse: 0.14583 | valid_rmsle: 0.08033 | valid_mae: 0.30664 | valid_rmse: 0.38204 | valid_mse: 0.14595 |  0:00:05s\n",
      "epoch 4  | loss: 0.11478 | train_rmsle: 0.0657  | train_mae: 0.2798  | train_rmse: 0.35083 | train_mse: 0.12308 | valid_rmsle: 0.06471 | valid_mae: 0.2762  | valid_rmse: 0.34809 | valid_mse: 0.12116 |  0:00:07s\n",
      "epoch 5  | loss: 0.11121 | train_rmsle: 0.05961 | train_mae: 0.25979 | train_rmse: 0.34053 | train_mse: 0.11596 | valid_rmsle: 0.05839 | valid_mae: 0.2566  | valid_rmse: 0.33742 | valid_mse: 0.11385 |  0:00:08s\n",
      "epoch 6  | loss: 0.11029 | train_rmsle: 0.06378 | train_mae: 0.25423 | train_rmse: 0.35531 | train_mse: 0.12624 | valid_rmsle: 0.06213 | valid_mae: 0.25054 | valid_rmse: 0.35461 | valid_mse: 0.12575 |  0:00:09s\n",
      "epoch 7  | loss: 0.11003 | train_rmsle: 0.057   | train_mae: 0.25219 | train_rmse: 0.34344 | train_mse: 0.11795 | valid_rmsle: 0.05512 | valid_mae: 0.2475  | valid_rmse: 0.33776 | valid_mse: 0.11408 |  0:00:11s\n",
      "epoch 8  | loss: 0.10988 | train_rmsle: 0.05512 | train_mae: 0.24226 | train_rmse: 0.33481 | train_mse: 0.1121  | valid_rmsle: 0.05343 | valid_mae: 0.23781 | valid_rmse: 0.32937 | valid_mse: 0.10848 |  0:00:12s\n",
      "epoch 9  | loss: 0.10829 | train_rmsle: 0.05693 | train_mae: 0.24596 | train_rmse: 0.33503 | train_mse: 0.11225 | valid_rmsle: 0.05497 | valid_mae: 0.24018 | valid_rmse: 0.32932 | valid_mse: 0.10845 |  0:00:13s\n",
      "epoch 10 | loss: 0.10833 | train_rmsle: 0.05375 | train_mae: 0.23563 | train_rmse: 0.33305 | train_mse: 0.11092 | valid_rmsle: 0.05185 | valid_mae: 0.23062 | valid_rmse: 0.32643 | valid_mse: 0.10656 |  0:00:15s\n",
      "epoch 11 | loss: 0.10764 | train_rmsle: 0.0526  | train_mae: 0.22899 | train_rmse: 0.33287 | train_mse: 0.1108  | valid_rmsle: 0.05076 | valid_mae: 0.22411 | valid_rmse: 0.3265  | valid_mse: 0.1066  |  0:00:16s\n",
      "epoch 12 | loss: 0.10698 | train_rmsle: 0.05624 | train_mae: 0.23993 | train_rmse: 0.33312 | train_mse: 0.11097 | valid_rmsle: 0.05461 | valid_mae: 0.2353  | valid_rmse: 0.32751 | valid_mse: 0.10726 |  0:00:18s\n",
      "epoch 13 | loss: 0.10679 | train_rmsle: 0.05276 | train_mae: 0.22877 | train_rmse: 0.33555 | train_mse: 0.1126  | valid_rmsle: 0.05128 | valid_mae: 0.22473 | valid_rmse: 0.33015 | valid_mse: 0.109   |  0:00:19s\n",
      "epoch 14 | loss: 0.10577 | train_rmsle: 0.05284 | train_mae: 0.22906 | train_rmse: 0.32791 | train_mse: 0.10752 | valid_rmsle: 0.05154 | valid_mae: 0.22525 | valid_rmse: 0.32335 | valid_mse: 0.10455 |  0:00:20s\n",
      "epoch 15 | loss: 0.10497 | train_rmsle: 0.05111 | train_mae: 0.22535 | train_rmse: 0.32761 | train_mse: 0.10733 | valid_rmsle: 0.05009 | valid_mae: 0.22221 | valid_rmse: 0.32379 | valid_mse: 0.10484 |  0:00:22s\n",
      "epoch 16 | loss: 0.10512 | train_rmsle: 0.05156 | train_mae: 0.22343 | train_rmse: 0.32994 | train_mse: 0.10886 | valid_rmsle: 0.05017 | valid_mae: 0.21935 | valid_rmse: 0.32502 | valid_mse: 0.10564 |  0:00:23s\n",
      "epoch 17 | loss: 0.10614 | train_rmsle: 0.05414 | train_mae: 0.22927 | train_rmse: 0.32792 | train_mse: 0.10753 | valid_rmsle: 0.05246 | valid_mae: 0.22524 | valid_rmse: 0.32246 | valid_mse: 0.10398 |  0:00:25s\n",
      "epoch 18 | loss: 0.1028  | train_rmsle: 0.05195 | train_mae: 0.22859 | train_rmse: 0.32487 | train_mse: 0.10554 | valid_rmsle: 0.05053 | valid_mae: 0.22437 | valid_rmse: 0.32036 | valid_mse: 0.10263 |  0:00:26s\n",
      "epoch 19 | loss: 0.10342 | train_rmsle: 0.05142 | train_mae: 0.22324 | train_rmse: 0.32451 | train_mse: 0.10531 | valid_rmsle: 0.04992 | valid_mae: 0.21936 | valid_rmse: 0.31937 | valid_mse: 0.10199 |  0:00:27s\n",
      "epoch 20 | loss: 0.10317 | train_rmsle: 0.05125 | train_mae: 0.22449 | train_rmse: 0.32393 | train_mse: 0.10493 | valid_rmsle: 0.04942 | valid_mae: 0.22011 | valid_rmse: 0.31785 | valid_mse: 0.10103 |  0:00:29s\n",
      "epoch 21 | loss: 0.10194 | train_rmsle: 0.0503  | train_mae: 0.21491 | train_rmse: 0.32128 | train_mse: 0.10322 | valid_rmsle: 0.0485  | valid_mae: 0.21037 | valid_rmse: 0.31517 | valid_mse: 0.09933 |  0:00:30s\n",
      "epoch 22 | loss: 0.10324 | train_rmsle: 0.05067 | train_mae: 0.22173 | train_rmse: 0.3247  | train_mse: 0.10543 | valid_rmsle: 0.04861 | valid_mae: 0.21679 | valid_rmse: 0.31732 | valid_mse: 0.10069 |  0:00:31s\n",
      "epoch 23 | loss: 0.10248 | train_rmsle: 0.05005 | train_mae: 0.21331 | train_rmse: 0.32139 | train_mse: 0.10329 | valid_rmsle: 0.04804 | valid_mae: 0.20833 | valid_rmse: 0.31477 | valid_mse: 0.09908 |  0:00:33s\n",
      "epoch 24 | loss: 0.10151 | train_rmsle: 0.04969 | train_mae: 0.215   | train_rmse: 0.32014 | train_mse: 0.10249 | valid_rmsle: 0.04756 | valid_mae: 0.20974 | valid_rmse: 0.31347 | valid_mse: 0.09826 |  0:00:34s\n",
      "epoch 25 | loss: 0.10211 | train_rmsle: 0.04935 | train_mae: 0.21158 | train_rmse: 0.3224  | train_mse: 0.10394 | valid_rmsle: 0.04687 | valid_mae: 0.20514 | valid_rmse: 0.31442 | valid_mse: 0.09886 |  0:00:36s\n",
      "epoch 26 | loss: 0.10208 | train_rmsle: 0.04962 | train_mae: 0.21607 | train_rmse: 0.31855 | train_mse: 0.10148 | valid_rmsle: 0.04761 | valid_mae: 0.21083 | valid_rmse: 0.31159 | valid_mse: 0.09709 |  0:00:37s\n",
      "epoch 27 | loss: 0.10094 | train_rmsle: 0.05275 | train_mae: 0.22095 | train_rmse: 0.32081 | train_mse: 0.10292 | valid_rmsle: 0.05063 | valid_mae: 0.2158  | valid_rmse: 0.3131  | valid_mse: 0.09803 |  0:00:39s\n",
      "epoch 28 | loss: 0.10124 | train_rmsle: 0.04891 | train_mae: 0.2123  | train_rmse: 0.32237 | train_mse: 0.10392 | valid_rmsle: 0.04737 | valid_mae: 0.20892 | valid_rmse: 0.31726 | valid_mse: 0.10065 |  0:00:40s\n",
      "epoch 29 | loss: 0.10122 | train_rmsle: 0.04956 | train_mae: 0.2099  | train_rmse: 0.31865 | train_mse: 0.10154 | valid_rmsle: 0.04759 | valid_mae: 0.20479 | valid_rmse: 0.31198 | valid_mse: 0.09733 |  0:00:41s\n",
      "epoch 30 | loss: 0.10027 | train_rmsle: 0.0496  | train_mae: 0.21145 | train_rmse: 0.31576 | train_mse: 0.0997  | valid_rmsle: 0.04841 | valid_mae: 0.20842 | valid_rmse: 0.31175 | valid_mse: 0.09719 |  0:00:43s\n",
      "epoch 31 | loss: 0.10034 | train_rmsle: 0.05005 | train_mae: 0.21385 | train_rmse: 0.3167  | train_mse: 0.1003  | valid_rmsle: 0.049   | valid_mae: 0.21084 | valid_rmse: 0.31334 | valid_mse: 0.09818 |  0:00:44s\n",
      "epoch 32 | loss: 0.09935 | train_rmsle: 0.04969 | train_mae: 0.21522 | train_rmse: 0.31826 | train_mse: 0.10129 | valid_rmsle: 0.04807 | valid_mae: 0.2116  | valid_rmse: 0.31319 | valid_mse: 0.09809 |  0:00:46s\n",
      "epoch 33 | loss: 0.09984 | train_rmsle: 0.04837 | train_mae: 0.21348 | train_rmse: 0.31861 | train_mse: 0.10152 | valid_rmsle: 0.04715 | valid_mae: 0.21017 | valid_rmse: 0.31453 | valid_mse: 0.09893 |  0:00:47s\n",
      "epoch 34 | loss: 0.09977 | train_rmsle: 0.04906 | train_mae: 0.2057  | train_rmse: 0.31576 | train_mse: 0.09971 | valid_rmsle: 0.04742 | valid_mae: 0.20152 | valid_rmse: 0.31029 | valid_mse: 0.09628 |  0:00:48s\n",
      "epoch 35 | loss: 0.09988 | train_rmsle: 0.04863 | train_mae: 0.20762 | train_rmse: 0.31561 | train_mse: 0.09961 | valid_rmsle: 0.04672 | valid_mae: 0.20284 | valid_rmse: 0.30913 | valid_mse: 0.09556 |  0:00:50s\n",
      "epoch 36 | loss: 0.10015 | train_rmsle: 0.04951 | train_mae: 0.21187 | train_rmse: 0.31592 | train_mse: 0.09981 | valid_rmsle: 0.04835 | valid_mae: 0.20902 | valid_rmse: 0.31217 | valid_mse: 0.09745 |  0:00:51s\n",
      "epoch 37 | loss: 0.10147 | train_rmsle: 0.05169 | train_mae: 0.2214  | train_rmse: 0.32128 | train_mse: 0.10322 | valid_rmsle: 0.04984 | valid_mae: 0.21678 | valid_rmse: 0.31476 | valid_mse: 0.09908 |  0:00:53s\n",
      "epoch 38 | loss: 0.10198 | train_rmsle: 0.04936 | train_mae: 0.21867 | train_rmse: 0.32333 | train_mse: 0.10454 | valid_rmsle: 0.04824 | valid_mae: 0.21547 | valid_rmse: 0.31959 | valid_mse: 0.10214 |  0:00:54s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 39 | loss: 0.10109 | train_rmsle: 0.05026 | train_mae: 0.21729 | train_rmse: 0.31905 | train_mse: 0.1018  | valid_rmsle: 0.04844 | valid_mae: 0.21238 | valid_rmse: 0.31259 | valid_mse: 0.09771 |  0:00:55s\n",
      "epoch 40 | loss: 0.1015  | train_rmsle: 0.05009 | train_mae: 0.21863 | train_rmse: 0.31728 | train_mse: 0.10067 | valid_rmsle: 0.04864 | valid_mae: 0.21431 | valid_rmse: 0.31218 | valid_mse: 0.09746 |  0:00:57s\n",
      "epoch 41 | loss: 0.10149 | train_rmsle: 0.04867 | train_mae: 0.21139 | train_rmse: 0.32118 | train_mse: 0.10316 | valid_rmsle: 0.04726 | valid_mae: 0.20743 | valid_rmse: 0.31627 | valid_mse: 0.10002 |  0:00:58s\n",
      "epoch 42 | loss: 0.10082 | train_rmsle: 0.0507  | train_mae: 0.21215 | train_rmse: 0.31643 | train_mse: 0.10013 | valid_rmsle: 0.04898 | valid_mae: 0.20688 | valid_rmse: 0.31061 | valid_mse: 0.09648 |  0:01:00s\n",
      "epoch 43 | loss: 0.1012  | train_rmsle: 0.05138 | train_mae: 0.21497 | train_rmse: 0.31801 | train_mse: 0.10113 | valid_rmsle: 0.04964 | valid_mae: 0.21004 | valid_rmse: 0.31243 | valid_mse: 0.09761 |  0:01:01s\n",
      "epoch 44 | loss: 0.09969 | train_rmsle: 0.05366 | train_mae: 0.21694 | train_rmse: 0.32127 | train_mse: 0.10321 | valid_rmsle: 0.05198 | valid_mae: 0.21147 | valid_rmse: 0.31592 | valid_mse: 0.09981 |  0:01:02s\n",
      "epoch 45 | loss: 0.09983 | train_rmsle: 0.04806 | train_mae: 0.20948 | train_rmse: 0.3201  | train_mse: 0.10246 | valid_rmsle: 0.04689 | valid_mae: 0.20623 | valid_rmse: 0.31651 | valid_mse: 0.10018 |  0:01:04s\n",
      "epoch 46 | loss: 0.09854 | train_rmsle: 0.04937 | train_mae: 0.20906 | train_rmse: 0.31537 | train_mse: 0.09946 | valid_rmsle: 0.04784 | valid_mae: 0.20508 | valid_rmse: 0.31005 | valid_mse: 0.09613 |  0:01:05s\n",
      "epoch 47 | loss: 0.09865 | train_rmsle: 0.04765 | train_mae: 0.20356 | train_rmse: 0.31644 | train_mse: 0.10013 | valid_rmsle: 0.0461  | valid_mae: 0.19918 | valid_rmse: 0.31127 | valid_mse: 0.09689 |  0:01:07s\n",
      "epoch 48 | loss: 0.09813 | train_rmsle: 0.04759 | train_mae: 0.2065  | train_rmse: 0.31669 | train_mse: 0.10029 | valid_rmsle: 0.04641 | valid_mae: 0.20337 | valid_rmse: 0.31259 | valid_mse: 0.09771 |  0:01:08s\n",
      "epoch 49 | loss: 0.09894 | train_rmsle: 0.04811 | train_mae: 0.21058 | train_rmse: 0.31469 | train_mse: 0.09903 | valid_rmsle: 0.04708 | valid_mae: 0.20755 | valid_rmse: 0.31112 | valid_mse: 0.0968  |  0:01:09s\n",
      "epoch 50 | loss: 0.09902 | train_rmsle: 0.04838 | train_mae: 0.20795 | train_rmse: 0.31515 | train_mse: 0.09932 | valid_rmsle: 0.04738 | valid_mae: 0.20433 | valid_rmse: 0.31136 | valid_mse: 0.09695 |  0:01:11s\n",
      "epoch 51 | loss: 0.09911 | train_rmsle: 0.0484  | train_mae: 0.20765 | train_rmse: 0.31505 | train_mse: 0.09925 | valid_rmsle: 0.04698 | valid_mae: 0.20378 | valid_rmse: 0.31041 | valid_mse: 0.09635 |  0:01:12s\n",
      "epoch 52 | loss: 0.09917 | train_rmsle: 0.04775 | train_mae: 0.20601 | train_rmse: 0.3163  | train_mse: 0.10005 | valid_rmsle: 0.04622 | valid_mae: 0.20242 | valid_rmse: 0.31109 | valid_mse: 0.09678 |  0:01:14s\n",
      "epoch 53 | loss: 0.0982  | train_rmsle: 0.04794 | train_mae: 0.20522 | train_rmse: 0.31544 | train_mse: 0.0995  | valid_rmsle: 0.04642 | valid_mae: 0.20135 | valid_rmse: 0.31003 | valid_mse: 0.09612 |  0:01:15s\n",
      "epoch 54 | loss: 0.09837 | train_rmsle: 0.04802 | train_mae: 0.20703 | train_rmse: 0.31501 | train_mse: 0.09923 | valid_rmsle: 0.04645 | valid_mae: 0.20307 | valid_rmse: 0.30927 | valid_mse: 0.09565 |  0:01:16s\n",
      "epoch 55 | loss: 0.09796 | train_rmsle: 0.04762 | train_mae: 0.2029  | train_rmse: 0.31699 | train_mse: 0.10048 | valid_rmsle: 0.04655 | valid_mae: 0.20033 | valid_rmse: 0.31329 | valid_mse: 0.09815 |  0:01:18s\n",
      "epoch 56 | loss: 0.09951 | train_rmsle: 0.04796 | train_mae: 0.20457 | train_rmse: 0.31803 | train_mse: 0.10114 | valid_rmsle: 0.0467  | valid_mae: 0.20155 | valid_rmse: 0.31322 | valid_mse: 0.09811 |  0:01:19s\n",
      "epoch 57 | loss: 0.09864 | train_rmsle: 0.04795 | train_mae: 0.20608 | train_rmse: 0.31984 | train_mse: 0.1023  | valid_rmsle: 0.04688 | valid_mae: 0.20307 | valid_rmse: 0.31594 | valid_mse: 0.09982 |  0:01:21s\n",
      "epoch 58 | loss: 0.09906 | train_rmsle: 0.04846 | train_mae: 0.20418 | train_rmse: 0.31676 | train_mse: 0.10034 | valid_rmsle: 0.04721 | valid_mae: 0.20046 | valid_rmse: 0.31195 | valid_mse: 0.09732 |  0:01:22s\n",
      "epoch 59 | loss: 0.09942 | train_rmsle: 0.04874 | train_mae: 0.20833 | train_rmse: 0.31395 | train_mse: 0.09856 | valid_rmsle: 0.04808 | valid_mae: 0.20561 | valid_rmse: 0.31126 | valid_mse: 0.09689 |  0:01:23s\n",
      "epoch 60 | loss: 0.09816 | train_rmsle: 0.04848 | train_mae: 0.20814 | train_rmse: 0.3157  | train_mse: 0.09967 | valid_rmsle: 0.04738 | valid_mae: 0.20452 | valid_rmse: 0.31145 | valid_mse: 0.097   |  0:01:25s\n",
      "epoch 61 | loss: 0.09783 | train_rmsle: 0.04989 | train_mae: 0.20517 | train_rmse: 0.31546 | train_mse: 0.09951 | valid_rmsle: 0.04939 | valid_mae: 0.20392 | valid_rmse: 0.31329 | valid_mse: 0.09815 |  0:01:26s\n",
      "epoch 62 | loss: 0.09757 | train_rmsle: 0.0489  | train_mae: 0.20754 | train_rmse: 0.31342 | train_mse: 0.09823 | valid_rmsle: 0.04812 | valid_mae: 0.20535 | valid_rmse: 0.31025 | valid_mse: 0.09626 |  0:01:28s\n",
      "epoch 63 | loss: 0.09733 | train_rmsle: 0.05012 | train_mae: 0.20823 | train_rmse: 0.31422 | train_mse: 0.09873 | valid_rmsle: 0.04929 | valid_mae: 0.20605 | valid_rmse: 0.31077 | valid_mse: 0.09658 |  0:01:29s\n",
      "epoch 64 | loss: 0.09746 | train_rmsle: 0.04716 | train_mae: 0.20427 | train_rmse: 0.31561 | train_mse: 0.09961 | valid_rmsle: 0.04616 | valid_mae: 0.20203 | valid_rmse: 0.31183 | valid_mse: 0.09724 |  0:01:30s\n",
      "epoch 65 | loss: 0.09895 | train_rmsle: 0.04738 | train_mae: 0.2032  | train_rmse: 0.31327 | train_mse: 0.09814 | valid_rmsle: 0.0465  | valid_mae: 0.20116 | valid_rmse: 0.31021 | valid_mse: 0.09623 |  0:01:32s\n",
      "epoch 66 | loss: 0.09839 | train_rmsle: 0.04817 | train_mae: 0.21045 | train_rmse: 0.31406 | train_mse: 0.09864 | valid_rmsle: 0.04732 | valid_mae: 0.20821 | valid_rmse: 0.31109 | valid_mse: 0.09677 |  0:01:33s\n",
      "epoch 67 | loss: 0.09867 | train_rmsle: 0.04858 | train_mae: 0.20448 | train_rmse: 0.31458 | train_mse: 0.09896 | valid_rmsle: 0.04756 | valid_mae: 0.20188 | valid_rmse: 0.31064 | valid_mse: 0.0965  |  0:01:34s\n",
      "epoch 68 | loss: 0.0975  | train_rmsle: 0.04824 | train_mae: 0.20539 | train_rmse: 0.31288 | train_mse: 0.09789 | valid_rmsle: 0.04716 | valid_mae: 0.203   | valid_rmse: 0.30895 | valid_mse: 0.09545 |  0:01:36s\n",
      "epoch 69 | loss: 0.09763 | train_rmsle: 0.04792 | train_mae: 0.20796 | train_rmse: 0.31485 | train_mse: 0.09913 | valid_rmsle: 0.04667 | valid_mae: 0.20488 | valid_rmse: 0.31019 | valid_mse: 0.09622 |  0:01:37s\n",
      "epoch 70 | loss: 0.09723 | train_rmsle: 0.04897 | train_mae: 0.2058  | train_rmse: 0.31261 | train_mse: 0.09773 | valid_rmsle: 0.04791 | valid_mae: 0.2035  | valid_rmse: 0.30854 | valid_mse: 0.0952  |  0:01:39s\n",
      "epoch 71 | loss: 0.09762 | train_rmsle: 0.04741 | train_mae: 0.20414 | train_rmse: 0.31256 | train_mse: 0.0977  | valid_rmsle: 0.04639 | valid_mae: 0.20201 | valid_rmse: 0.30874 | valid_mse: 0.09532 |  0:01:40s\n",
      "epoch 72 | loss: 0.0971  | train_rmsle: 0.04862 | train_mae: 0.20357 | train_rmse: 0.31456 | train_mse: 0.09895 | valid_rmsle: 0.04736 | valid_mae: 0.20057 | valid_rmse: 0.30995 | valid_mse: 0.09607 |  0:01:41s\n",
      "epoch 73 | loss: 0.09689 | train_rmsle: 0.04786 | train_mae: 0.21148 | train_rmse: 0.31723 | train_mse: 0.10063 | valid_rmsle: 0.04678 | valid_mae: 0.20891 | valid_rmse: 0.31349 | valid_mse: 0.09828 |  0:01:43s\n",
      "epoch 74 | loss: 0.09743 | train_rmsle: 0.04749 | train_mae: 0.20384 | train_rmse: 0.31325 | train_mse: 0.09812 | valid_rmsle: 0.04617 | valid_mae: 0.20132 | valid_rmse: 0.30846 | valid_mse: 0.09515 |  0:01:44s\n",
      "epoch 75 | loss: 0.0972  | train_rmsle: 0.04803 | train_mae: 0.20561 | train_rmse: 0.31244 | train_mse: 0.09762 | valid_rmsle: 0.04672 | valid_mae: 0.20277 | valid_rmse: 0.30754 | valid_mse: 0.09458 |  0:01:45s\n",
      "epoch 76 | loss: 0.09595 | train_rmsle: 0.0478  | train_mae: 0.20355 | train_rmse: 0.31466 | train_mse: 0.09901 | valid_rmsle: 0.04658 | valid_mae: 0.20084 | valid_rmse: 0.31016 | valid_mse: 0.0962  |  0:01:47s\n",
      "epoch 77 | loss: 0.09632 | train_rmsle: 0.0472  | train_mae: 0.20232 | train_rmse: 0.31319 | train_mse: 0.09809 | valid_rmsle: 0.04605 | valid_mae: 0.20031 | valid_rmse: 0.30921 | valid_mse: 0.09561 |  0:01:48s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 78 | loss: 0.09692 | train_rmsle: 0.04901 | train_mae: 0.20755 | train_rmse: 0.31433 | train_mse: 0.0988  | valid_rmsle: 0.04771 | valid_mae: 0.20417 | valid_rmse: 0.30959 | valid_mse: 0.09585 |  0:01:50s\n",
      "epoch 79 | loss: 0.09665 | train_rmsle: 0.04741 | train_mae: 0.2011  | train_rmse: 0.31131 | train_mse: 0.09691 | valid_rmsle: 0.04623 | valid_mae: 0.19879 | valid_rmse: 0.30713 | valid_mse: 0.09433 |  0:01:51s\n",
      "epoch 80 | loss: 0.09707 | train_rmsle: 0.04718 | train_mae: 0.20445 | train_rmse: 0.31355 | train_mse: 0.09831 | valid_rmsle: 0.04589 | valid_mae: 0.20147 | valid_rmse: 0.30907 | valid_mse: 0.09552 |  0:01:52s\n",
      "epoch 81 | loss: 0.09612 | train_rmsle: 0.04716 | train_mae: 0.20586 | train_rmse: 0.31175 | train_mse: 0.09719 | valid_rmsle: 0.04576 | valid_mae: 0.20278 | valid_rmse: 0.30691 | valid_mse: 0.09419 |  0:01:54s\n",
      "epoch 82 | loss: 0.09604 | train_rmsle: 0.04763 | train_mae: 0.20646 | train_rmse: 0.31163 | train_mse: 0.09711 | valid_rmsle: 0.04651 | valid_mae: 0.20362 | valid_rmse: 0.30755 | valid_mse: 0.09459 |  0:01:55s\n",
      "epoch 83 | loss: 0.09563 | train_rmsle: 0.04713 | train_mae: 0.20618 | train_rmse: 0.31666 | train_mse: 0.10027 | valid_rmsle: 0.04591 | valid_mae: 0.20295 | valid_rmse: 0.3122  | valid_mse: 0.09747 |  0:01:56s\n",
      "epoch 84 | loss: 0.09528 | train_rmsle: 0.0471  | train_mae: 0.20568 | train_rmse: 0.31303 | train_mse: 0.09799 | valid_rmsle: 0.04643 | valid_mae: 0.20403 | valid_rmse: 0.31028 | valid_mse: 0.09627 |  0:01:58s\n",
      "epoch 85 | loss: 0.09574 | train_rmsle: 0.04854 | train_mae: 0.2081  | train_rmse: 0.31425 | train_mse: 0.09875 | valid_rmsle: 0.04779 | valid_mae: 0.20611 | valid_rmse: 0.31135 | valid_mse: 0.09694 |  0:01:59s\n",
      "epoch 86 | loss: 0.09567 | train_rmsle: 0.04867 | train_mae: 0.20816 | train_rmse: 0.31457 | train_mse: 0.09895 | valid_rmsle: 0.0474  | valid_mae: 0.20504 | valid_rmse: 0.30997 | valid_mse: 0.09608 |  0:02:01s\n",
      "epoch 87 | loss: 0.09636 | train_rmsle: 0.04716 | train_mae: 0.20594 | train_rmse: 0.31315 | train_mse: 0.09806 | valid_rmsle: 0.04654 | valid_mae: 0.20452 | valid_rmse: 0.31063 | valid_mse: 0.09649 |  0:02:02s\n",
      "epoch 88 | loss: 0.09591 | train_rmsle: 0.04893 | train_mae: 0.20342 | train_rmse: 0.31486 | train_mse: 0.09914 | valid_rmsle: 0.04787 | valid_mae: 0.20101 | valid_rmse: 0.31063 | valid_mse: 0.09649 |  0:02:03s\n",
      "epoch 89 | loss: 0.09616 | train_rmsle: 0.04726 | train_mae: 0.2021  | train_rmse: 0.3109  | train_mse: 0.09666 | valid_rmsle: 0.04637 | valid_mae: 0.20035 | valid_rmse: 0.30758 | valid_mse: 0.09461 |  0:02:05s\n",
      "epoch 90 | loss: 0.09552 | train_rmsle: 0.04688 | train_mae: 0.20108 | train_rmse: 0.31004 | train_mse: 0.09612 | valid_rmsle: 0.04626 | valid_mae: 0.19981 | valid_rmse: 0.30742 | valid_mse: 0.09451 |  0:02:06s\n",
      "epoch 91 | loss: 0.09573 | train_rmsle: 0.04755 | train_mae: 0.20916 | train_rmse: 0.31493 | train_mse: 0.09918 | valid_rmsle: 0.04691 | valid_mae: 0.20723 | valid_rmse: 0.31232 | valid_mse: 0.09754 |  0:02:08s\n",
      "epoch 92 | loss: 0.09485 | train_rmsle: 0.04686 | train_mae: 0.20371 | train_rmse: 0.31634 | train_mse: 0.10007 | valid_rmsle: 0.04612 | valid_mae: 0.20201 | valid_rmse: 0.31364 | valid_mse: 0.09837 |  0:02:09s\n",
      "epoch 93 | loss: 0.09578 | train_rmsle: 0.04734 | train_mae: 0.20391 | train_rmse: 0.31296 | train_mse: 0.09794 | valid_rmsle: 0.0466  | valid_mae: 0.20216 | valid_rmse: 0.31012 | valid_mse: 0.09617 |  0:02:10s\n",
      "epoch 94 | loss: 0.09567 | train_rmsle: 0.04701 | train_mae: 0.20346 | train_rmse: 0.31216 | train_mse: 0.09744 | valid_rmsle: 0.0463  | valid_mae: 0.20177 | valid_rmse: 0.3092  | valid_mse: 0.0956  |  0:02:12s\n",
      "epoch 95 | loss: 0.09615 | train_rmsle: 0.04657 | train_mae: 0.19859 | train_rmse: 0.31408 | train_mse: 0.09864 | valid_rmsle: 0.04581 | valid_mae: 0.19735 | valid_rmse: 0.31106 | valid_mse: 0.09676 |  0:02:13s\n",
      "epoch 96 | loss: 0.09594 | train_rmsle: 0.04694 | train_mae: 0.20517 | train_rmse: 0.31133 | train_mse: 0.09692 | valid_rmsle: 0.04651 | valid_mae: 0.20409 | valid_rmse: 0.30935 | valid_mse: 0.0957  |  0:02:14s\n",
      "epoch 97 | loss: 0.09541 | train_rmsle: 0.04739 | train_mae: 0.20526 | train_rmse: 0.31208 | train_mse: 0.09739 | valid_rmsle: 0.04679 | valid_mae: 0.2037  | valid_rmse: 0.30957 | valid_mse: 0.09583 |  0:02:16s\n",
      "epoch 98 | loss: 0.09606 | train_rmsle: 0.04706 | train_mae: 0.20423 | train_rmse: 0.31524 | train_mse: 0.09938 | valid_rmsle: 0.04626 | valid_mae: 0.20216 | valid_rmse: 0.3122  | valid_mse: 0.09747 |  0:02:17s\n",
      "epoch 99 | loss: 0.09641 | train_rmsle: 0.04973 | train_mae: 0.20321 | train_rmse: 0.31253 | train_mse: 0.09767 | valid_rmsle: 0.04895 | valid_mae: 0.2007  | valid_rmse: 0.30926 | valid_mse: 0.09564 |  0:02:18s\n",
      "Stop training because you reached max_epochs = 100 with best_epoch = 81 and best_valid_mse = 0.09419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/work/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n",
      "  warnings.warn(wrn_msg)\n"
     ]
    }
   ],
   "source": [
    "clf.fit(\n",
    "    X_train=X_train, y_train=y_train,\n",
    "    eval_set=[(X_train, y_train), (X_valid, y_valid)],\n",
    "    eval_name=['train', 'valid'],\n",
    "    eval_metric=['rmsle', 'mae', 'rmse', 'mse'],\n",
    "    max_epochs=max_epochs,\n",
    "    patience=50,\n",
    "    batch_size=1024, virtual_batch_size=128,\n",
    "    num_workers=0,\n",
    "    drop_last=False,\n",
    "    augmentations=aug, #aug\n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deprecated : best model is automatically loaded at end of fit\n",
    "# clf.load_best_model()\n",
    "\n",
    "preds = clf.predict(X_test)\n",
    "\n",
    "y_true = y_test\n",
    "\n",
    "test_score = mean_squared_error(y_pred=preds, y_true=y_true)\n",
    "\n",
    "print(f\"BEST VALID SCORE FOR {dataset_name} : {clf.best_cost}\")\n",
    "print(f\"FINAL TEST SCORE FOR {dataset_name} : {test_score}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save model and load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tabnet model\n",
    "saving_path_name = \"./tabnet_model_test_1\"\n",
    "saved_filepath = clf.save_model(saving_path_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define new model with basic parameters and load state dict weights\n",
    "loaded_clf = TabNetRegressor()\n",
    "loaded_clf.load_model(saved_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_preds = loaded_clf.predict(X_test)\n",
    "loaded_test_mse = mean_squared_error(loaded_preds, y_test)\n",
    "\n",
    "print(f\"FINAL TEST SCORE FOR {dataset_name} : {loaded_test_mse}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(test_score == loaded_test_mse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Global explainability : feat importance summing to 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.feature_importances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Local explainability and masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explain_matrix, masks = clf.explain(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(1, 3, figsize=(20,20))\n",
    "\n",
    "for i in range(3):\n",
    "    axs[i].imshow(masks[i][:50])\n",
    "    axs[i].set_title(f\"mask {i}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBRegressor\n",
    "\n",
    "clf_xgb = XGBRegressor(max_depth=8,\n",
    "    learning_rate=0.1,\n",
    "    n_estimators=1000,\n",
    "    verbosity=0,\n",
    "    silent=None,\n",
    "    objective='reg:linear',\n",
    "    booster='gbtree',\n",
    "    n_jobs=-1,\n",
    "    nthread=None,\n",
    "    gamma=0,\n",
    "    min_child_weight=1,\n",
    "    max_delta_step=0,\n",
    "    subsample=0.7,\n",
    "    colsample_bytree=1,\n",
    "    colsample_bylevel=1,\n",
    "    colsample_bynode=1,\n",
    "    reg_alpha=0,\n",
    "    reg_lambda=1,\n",
    "    scale_pos_weight=1,\n",
    "    base_score=0.5,\n",
    "    random_state=0,\n",
    "    seed=None,)\n",
    "\n",
    "clf_xgb.fit(X_train, y_train,\n",
    "        eval_set=[(X_valid, y_valid)],\n",
    "        early_stopping_rounds=40,\n",
    "        verbose=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = np.array(clf_xgb.predict(X_valid))\n",
    "valid_auc = mean_squared_error(y_pred=preds, y_true=y_valid)\n",
    "print(valid_auc)\n",
    "\n",
    "preds = np.array(clf_xgb.predict(X_test))\n",
    "test_auc = mean_squared_error(y_pred=preds, y_true=y_test)\n",
    "print(test_auc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}