model-01-predict-lgb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/lightgbm/__init__.py:45: FutureWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS will be built by the Apple Clang compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you won't need to install the gcc compiler anymore.\n",
      "Instead of that, you'll need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import pickle\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from utilities import cal_mape\n",
    "\n",
    "from utilities import cal_score, convert_types, create_dt_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'model-01-lgb'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_log_target = True\n",
    "is_per_area_target = True\n",
    "\n",
    "map_target = {\n",
    "    (True, True):   'log_per_area_total_price',\n",
    "    (True, False):  'log_total_price',\n",
    "    (False, True):  'per_area_total_price',\n",
    "    (False, False): 'total_price'\n",
    "}\n",
    "col_target = map_target[(is_log_target, is_per_area_target)]\n",
    "\n",
    "#cols_feats_add = ['encoded_latlon', 'building_per_land_area'] # add feats into model\n",
    "cols_num_add = ['building_per_land_area']\n",
    "cols_cat_add = ['encoded_latlon']\n",
    "\n",
    "#cols_feats_rm = [] # TODO: implement before using it\n",
    "#['day_in_year_txn_dt', 'day_in_year_building_complete_dt', 'dt_diff_building_complete_dt_txn_dt'] # dt num feats\n",
    "#['dow_txn_dt', 'dow_building_complete_dt', 'month_txn_dt', 'month_building_complete_dt',\n",
    "# 'quarter_txn_dt', 'quarter_building_complete_dt', 'year_txn_dt', 'year_building_complete_dt'] # dt cat feats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_processing(df, cols_num, cols_cat): #, cols_feats\n",
    "    # Convert types\n",
    "    df[cols_num] = df[cols_num].astype('float32')\n",
    "    \n",
    "    # generate target\n",
    "    if 'total_price' in df:\n",
    "        df['total_price'] = df['total_price'].astype('float32')\n",
    "        df['log_total_price'] = np.log1p(df['total_price'])\n",
    "        df['per_area_total_price'] = df['total_price'] / df['building_area']\n",
    "        df['log_per_area_total_price'] = np.log1p(df['per_area_total_price'])\n",
    "        df['target'] = df[col_target]\n",
    "\n",
    "    # generate lat/lon encoded\n",
    "    kmeans = pickle.load(open('./object_kmeans_k6_combined.pkl', 'rb'))\n",
    "    df['encoded_latlon'] = kmeans.predict(df[['lat','lon']])\n",
    "    \n",
    "    # generate dt feats\n",
    "    df = create_dt_feats(df, 'txn_dt')\n",
    "    df = create_dt_feats(df, 'building_complete_dt')\n",
    "    df['dt_diff_building_complete_dt_txn_dt'] = df['txn_dt'] - df['building_complete_dt']\n",
    "    \n",
    "    # generate building to land ratio\n",
    "    df['building_per_land_area'] = df['building_area'] / df['land_area']\n",
    " \n",
    "    #cols_feats = cols_num + cols_cat + cols_feats_add\n",
    "    \n",
    "    return df  #, cols_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from feature_engineering import CategoricalColumnsEncoder\n",
    "from feature_engineering import TargetMeanEncoding\n",
    "from feature_engineering import MultiLabelEncoding\n",
    "\n",
    "class FeatureEngineering():\n",
    "    def __init__(self, cols_num, cols_cat, cols_feats, col_target, n_ran_encode=4):\n",
    "        # cols\n",
    "        self.cols_num = cols_num\n",
    "        self.cols_cat = cols_cat\n",
    "        self.cols_feats = cols_feats\n",
    "        self.col_target = col_target\n",
    "        \n",
    "        # Numbers\n",
    "        self.n_ran_encode = n_ran_encode\n",
    "        \n",
    "        # encoders\n",
    "        #self.tar_encoder = TargetMeanEncoding(col_target = self.col_target)\n",
    "        #self.ran_encoder = MultiLabelEncoding(self.n_ran_encode)\n",
    "        self.cat_astyper = CategoricalColumnsEncoder(mode='pandas')\n",
    "        \n",
    "    def fit_transform(self, df):\n",
    "        df = df.copy()\n",
    "    \n",
    "#        encoded1 = self.tar_encoder.fit_transform(df[self.cols_cat], df[self.col_target])\n",
    "#        encoded2 = self.ran_encoder.fit_transform(df[self.cols_cat])\n",
    "        self.cat_astyper.fit_transform(df, self.cols_cat)    \n",
    "        return df\n",
    "#        return pd.concat([df, encoded1, encoded2], axis=1)\n",
    "    \n",
    "    def transform(self, df):\n",
    "        df = df.copy()\n",
    "#        encoded1 = self.tar_encoder.transform(df[self.cols_cat])\n",
    "#        encoded2 = self.ran_encoder.transform(df[self.cols_cat])     # TODO: add generated feats into cols_feats\n",
    "        self.cat_astyper.transform(df)\n",
    "        return df\n",
    "#        return pd.concat([df, encoded1, encoded2], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_processing(y_pred, df):\n",
    "    \"\"\"Args: \n",
    "          y_pred (1d np.array): raw prediction y\n",
    "          df (pd.Dataframe): input feature df\n",
    "       Return: np.array\n",
    "    \"\"\"\n",
    "    y_pred_final = y_pred.copy()\n",
    "    \n",
    "    if is_log_target:\n",
    "        y_pred_final = np.expm1(y_pred_final)\n",
    "        y_pred_final = np.clip(y_pred_final, 0, None)\n",
    "\n",
    "    if is_per_area_target:\n",
    "        y_pred_final = y_pred_final * df['building_area'].values\n",
    "        \n",
    "    return y_pred_final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_num = ['txn_dt', 'total_floor', 'building_complete_dt', 'parking_area', 'parking_price', 'txn_floor',\n",
    "            'land_area', 'building_area', 'lat', 'lon', 'village_income_median', 'town_population', 'town_area',\n",
    "            'town_population_density', 'doc_rate', 'master_rate', 'bachelor_rate', 'jobschool_rate',\n",
    "            'highschool_rate', 'junior_rate', 'elementary_rate', 'born_rate', 'death_rate', 'marriage_rate',\n",
    "            'divorce_rate', 'N_50', 'N_500', 'N_1000', 'N_5000', 'N_10000', 'I_10', 'I_50', 'I_100', 'I_250',\n",
    "            'I_500', 'I_1000', 'I_5000', 'I_10000', 'I_MIN', 'II_10', 'II_50', 'II_100', 'II_250', 'II_500',\n",
    "            'II_1000', 'II_5000', 'II_10000', 'II_MIN', 'III_10', 'III_50', 'III_100', 'III_250', 'III_500',\n",
    "            'III_1000', 'III_5000', 'III_10000', 'III_MIN', 'IV_10', 'IV_50', 'IV_100', 'IV_250', 'IV_500',\n",
    "            'IV_1000', 'IV_5000', 'IV_10000', 'IV_MIN', 'V_10', 'V_50', 'V_100', 'V_250', 'V_500', 'V_1000',\n",
    "            'V_5000', 'V_10000', 'V_MIN', 'VI_50', 'VI_100', 'VI_250', 'VI_500', 'VI_1000', 'VI_5000', 'VI_10000',\n",
    "            'VI_MIN', 'VII_10', 'VII_50', 'VII_100', 'VII_250', 'VII_500', 'VII_1000', 'VII_5000', 'VII_10000',\n",
    "            'VII_MIN', 'VIII_10', 'VIII_50', 'VIII_100', 'VIII_250', 'VIII_500', 'VIII_1000', 'VIII_5000',\n",
    "            'VIII_10000', 'VIII_MIN', 'IX_10', 'IX_50', 'IX_100', 'IX_250', 'IX_500', 'IX_1000', 'IX_5000',\n",
    "            'IX_10000', 'IX_MIN', 'X_10', 'X_50', 'X_100', 'X_250', 'X_500', 'X_1000', 'X_5000', 'X_10000', 'X_MIN',\n",
    "            'XI_10', 'XI_50', 'XI_100', 'XI_250', 'XI_500', 'XI_1000', 'XI_5000', 'XI_10000', 'XI_MIN', 'XII_10',\n",
    "            'XII_50', 'XII_100', 'XII_250', 'XII_500', 'XII_1000', 'XII_5000', 'XII_10000', 'XII_MIN', 'XIII_10',\n",
    "            'XIII_50', 'XIII_100', 'XIII_250', 'XIII_500', 'XIII_1000', 'XIII_5000', 'XIII_10000', 'XIII_MIN',\n",
    "            'XIV_10', 'XIV_50', 'XIV_100', 'XIV_250', 'XIV_500', 'XIV_1000', 'XIV_5000', 'XIV_10000', 'XIV_MIN']\n",
    "cols_cat = ['building_material', 'city', 'building_type', 'building_use', 'parking_way', 'town', 'village',\n",
    "            'I_index_50', 'I_index_500', 'I_index_1000', 'II_index_50', 'II_index_500', 'II_index_1000',\n",
    "            'III_index_50', 'III_index_500', 'III_index_1000', 'IV_index_50', 'IV_index_500', 'IV_index_1000',\n",
    "            'IV_index_5000', 'V_index_50', 'V_index_500', 'V_index_1000', 'VI_10', 'VI_index_50', 'VI_index_500',\n",
    "            'VI_index_1000', 'VII_index_50', 'VII_index_500', 'VII_index_1000', 'VIII_index_50', 'VIII_index_500',\n",
    "            'VIII_index_1000', 'IX_index_50', 'IX_index_500', 'IX_index_1000', 'IX_index_5000', 'X_index_50',\n",
    "            'X_index_500', 'X_index_1000', 'XI_index_50', 'XI_index_500', 'XI_index_1000', 'XI_index_5000',\n",
    "            'XI_index_10000', 'XII_index_50', 'XII_index_500', 'XII_index_1000', 'XIII_index_50', 'XIII_index_500',\n",
    "            'XIII_index_1000', 'XIII_index_5000', 'XIII_index_10000', 'XIV_index_50', 'XIV_index_500',\n",
    "            'XIV_index_1000']\n",
    "cols_feats = cols_num + cols_cat\n",
    "\n",
    "# informational cols\n",
    "cols_dt = ['txn_dt', 'building_complete_dt']\n",
    "cols_bin = ['I_index_50', 'I_index_500', 'I_index_1000', 'II_index_50', 'II_index_500', 'II_index_1000',\n",
    "            'III_index_50', 'III_index_500', 'III_index_1000', 'IV_index_50', 'IV_index_500', 'IV_index_1000',\n",
    "            'IV_index_5000', 'V_index_50', 'V_index_500', 'V_index_1000', 'VI_10', 'VI_index_50', 'VI_index_500',\n",
    "            'VI_index_1000', 'VII_index_50', 'VII_index_500', 'VII_index_1000', 'VIII_index_50', 'VIII_index_500',\n",
    "            'VIII_index_1000', 'IX_index_50', 'IX_index_500', 'IX_index_1000', 'IX_index_5000', 'X_index_50',\n",
    "            'X_index_500', 'X_index_1000', 'XI_index_50', 'XI_index_500', 'XI_index_1000', 'XI_index_5000',\n",
    "            'XI_index_10000', 'XII_index_50', 'XII_index_500', 'XII_index_1000', 'XIII_index_50', 'XIII_index_500',\n",
    "            'XIII_index_1000', 'XIII_index_5000', 'XIII_index_10000', 'XIV_index_50', 'XIV_index_500',\n",
    "            'XIV_index_1000']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('dataset/train.csv', dtype=object)\n",
    "test = pd.read_csv('dataset/test.csv', dtype=object)\n",
    "train.head(5)\n",
    "test.head(5)\n",
    "\n",
    "id_cv = []\n",
    "for i in range(5):\n",
    "    id_one_fold = pd.read_csv(f'dataset/id_cv{i}.csv', dtype=object)['building_id']\n",
    "    id_cv.append(id_one_fold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/base.py:251: UserWarning: Trying to unpickle estimator KMeans from version 0.20.3 when using version 0.20.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/base.py:251: UserWarning: Trying to unpickle estimator KMeans from version 0.20.3 when using version 0.20.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "train = pre_processing(train, cols_num, cols_cat)\n",
    "test = pre_processing(test, cols_num, cols_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_num = cols_num + [c for c in cols_num_add if c not in cols_num]\n",
    "cols_cat = cols_cat + [c for c in cols_cat_add if c not in cols_cat]\n",
    "cols_feats = cols_num + cols_cat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params\n",
    "params = {'task': 'train',\n",
    "          'boosting_type': 'gbdt',\n",
    "          'objective': 'mse',\n",
    "          'metric': 'mape',\n",
    "          'learning_rate': 0.01,\n",
    "          'max_depth': 16,\n",
    "          'num_leaves': 300,           # may reduce in dim-reduction exp\n",
    "          'feature_fraction': 0.5,\n",
    "          'min_data_in_leaf': 10,\n",
    "          'lambda_l1': 0.01,\n",
    "          'lambda_l2': 0.1,}\n",
    "\n",
    "lgb_other_params = {'num_boost_round': 100000,\n",
    "                    'verbose_eval': 2000,\n",
    "                    'early_stopping_rounds': 1000,}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### kfold cross-predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 1000 rounds.\n",
      "[2000]\tvalid_0's mape: 0.00945225\n",
      "[4000]\tvalid_0's mape: 0.00924236\n",
      "[6000]\tvalid_0's mape: 0.00918072\n",
      "[8000]\tvalid_0's mape: 0.00915603\n",
      "[10000]\tvalid_0's mape: 0.00914505\n",
      "[12000]\tvalid_0's mape: 0.00913853\n",
      "[14000]\tvalid_0's mape: 0.00913486\n",
      "[16000]\tvalid_0's mape: 0.009133\n",
      "[18000]\tvalid_0's mape: 0.00913159\n",
      "[20000]\tvalid_0's mape: 0.00913075\n",
      "[22000]\tvalid_0's mape: 0.00912996\n",
      "[24000]\tvalid_0's mape: 0.00912943\n",
      "[26000]\tvalid_0's mape: 0.00912902\n",
      "[28000]\tvalid_0's mape: 0.0091287\n",
      "[30000]\tvalid_0's mape: 0.00912845\n",
      "[32000]\tvalid_0's mape: 0.00912832\n",
      "Early stopping, best iteration is:\n",
      "[31888]\tvalid_0's mape: 0.00912832\n",
      "(Fold 0) score, mse, mae, mape: 5808.87519, 0.03617, 0.12143, 0.00913.\n",
      "Training until validation scores don't improve for 1000 rounds.\n",
      "[2000]\tvalid_0's mape: 0.00957961\n",
      "[4000]\tvalid_0's mape: 0.00933928\n",
      "[6000]\tvalid_0's mape: 0.00926507\n",
      "[8000]\tvalid_0's mape: 0.00923551\n",
      "[10000]\tvalid_0's mape: 0.00921955\n",
      "[12000]\tvalid_0's mape: 0.00921089\n",
      "[14000]\tvalid_0's mape: 0.00920569\n",
      "[16000]\tvalid_0's mape: 0.00920239\n",
      "[18000]\tvalid_0's mape: 0.00920017\n",
      "[20000]\tvalid_0's mape: 0.00919875\n",
      "[22000]\tvalid_0's mape: 0.00919755\n",
      "[24000]\tvalid_0's mape: 0.00919677\n",
      "[26000]\tvalid_0's mape: 0.00919615\n",
      "[28000]\tvalid_0's mape: 0.00919582\n",
      "[30000]\tvalid_0's mape: 0.00919551\n",
      "Early stopping, best iteration is:\n",
      "[30298]\tvalid_0's mape: 0.00919549\n",
      "(Fold 1) score, mse, mae, mape: 5907.87347, 0.03868, 0.12235, 0.00918.\n",
      "Training until validation scores don't improve for 1000 rounds.\n",
      "[2000]\tvalid_0's mape: 0.00955106\n",
      "[4000]\tvalid_0's mape: 0.00933349\n",
      "[6000]\tvalid_0's mape: 0.00927095\n",
      "[8000]\tvalid_0's mape: 0.00924714\n",
      "[10000]\tvalid_0's mape: 0.00923521\n",
      "[12000]\tvalid_0's mape: 0.00922898\n",
      "[14000]\tvalid_0's mape: 0.00922515\n",
      "[16000]\tvalid_0's mape: 0.00922251\n",
      "[18000]\tvalid_0's mape: 0.00922092\n",
      "[20000]\tvalid_0's mape: 0.00922002\n",
      "[22000]\tvalid_0's mape: 0.00921921\n",
      "[24000]\tvalid_0's mape: 0.00921847\n",
      "[26000]\tvalid_0's mape: 0.00921794\n",
      "[28000]\tvalid_0's mape: 0.00921751\n",
      "[30000]\tvalid_0's mape: 0.00921723\n",
      "Early stopping, best iteration is:\n",
      "[30226]\tvalid_0's mape: 0.0092172\n",
      "(Fold 2) score, mse, mae, mape: 5918.87087, 0.03891, 0.12235, 0.00920.\n",
      "Training until validation scores don't improve for 1000 rounds.\n",
      "[2000]\tvalid_0's mape: 0.00960475\n",
      "[4000]\tvalid_0's mape: 0.00936018\n",
      "[6000]\tvalid_0's mape: 0.0092857\n",
      "[8000]\tvalid_0's mape: 0.00925329\n",
      "[10000]\tvalid_0's mape: 0.00923851\n",
      "[12000]\tvalid_0's mape: 0.00923033\n",
      "[14000]\tvalid_0's mape: 0.00922496\n",
      "[16000]\tvalid_0's mape: 0.00922187\n",
      "[18000]\tvalid_0's mape: 0.00922006\n",
      "[20000]\tvalid_0's mape: 0.00921889\n",
      "[22000]\tvalid_0's mape: 0.00921807\n",
      "[24000]\tvalid_0's mape: 0.00921738\n",
      "[26000]\tvalid_0's mape: 0.0092169\n",
      "[28000]\tvalid_0's mape: 0.00921652\n",
      "[30000]\tvalid_0's mape: 0.00921627\n",
      "Early stopping, best iteration is:\n",
      "[30921]\tvalid_0's mape: 0.00921619\n",
      "(Fold 3) score, mse, mae, mape: 5874.87399, 0.03752, 0.12267, 0.00921.\n",
      "Training until validation scores don't improve for 1000 rounds.\n",
      "[2000]\tvalid_0's mape: 0.00947793\n",
      "[4000]\tvalid_0's mape: 0.00925785\n",
      "[6000]\tvalid_0's mape: 0.00919503\n",
      "[8000]\tvalid_0's mape: 0.00916778\n",
      "[10000]\tvalid_0's mape: 0.00915571\n",
      "[12000]\tvalid_0's mape: 0.00915014\n",
      "[14000]\tvalid_0's mape: 0.00914601\n",
      "[16000]\tvalid_0's mape: 0.00914414\n",
      "[18000]\tvalid_0's mape: 0.00914304\n",
      "[20000]\tvalid_0's mape: 0.00914206\n",
      "[22000]\tvalid_0's mape: 0.00914136\n",
      "[24000]\tvalid_0's mape: 0.00914093\n",
      "[26000]\tvalid_0's mape: 0.00914059\n",
      "[28000]\tvalid_0's mape: 0.00914033\n",
      "[30000]\tvalid_0's mape: 0.00914016\n",
      "Early stopping, best iteration is:\n",
      "[30332]\tvalid_0's mape: 0.00914014\n",
      "(Fold 4) score, mse, mae, mape: 5843.87177, 0.03728, 0.12142, 0.00913.\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "\n",
    "predicts_val = []\n",
    "predicts_test = test[['building_id']].copy()\n",
    "best_iterations = []\n",
    "scores = []\n",
    "for i, id_fold in enumerate(id_cv):\n",
    "    #id_fold = id_fold['building_id']\n",
    "    # split train, val\n",
    "    df_train = train.loc[~train['building_id'].isin(id_fold)].copy()\n",
    "    df_val = train.loc[train['building_id'].isin(id_fold)].copy()\n",
    "    df_test = test.copy()\n",
    "    \n",
    "    # feat eng\n",
    "    feat_eng = FeatureEngineering(cols_num, cols_cat, cols_feats, col_target)\n",
    "    df_train = feat_eng.fit_transform(df_train)\n",
    "    df_val = feat_eng.transform(df_val)\n",
    "    df_test = feat_eng.transform(df_test)\n",
    "\n",
    "    # Construct lgb dataset\n",
    "    lgb_train = lgb.Dataset(df_train[cols_feats], df_train[col_target]).construct()\n",
    "    lgb_val = lgb.Dataset(df_val[cols_feats], df_val[col_target], reference=lgb_train).construct()\n",
    "        \n",
    "    # train model    \n",
    "    model = lgb.train(params, lgb_train, valid_sets=lgb_val, **lgb_other_params)\n",
    "    best_iterations.append(model.best_iteration)\n",
    "\n",
    "    # Predict on val set\n",
    "    y_pred = model.predict(df_val[cols_feats])\n",
    "    mse = mean_squared_error(df_val[col_target], y_pred)\n",
    "    mae = mean_absolute_error(df_val[col_target], y_pred)\n",
    "    mape = cal_mape(df_val[col_target], y_pred)\n",
    "    \n",
    "    y_pred_final = post_processing(y_pred, df_val)\n",
    "    score = cal_score(df_val['total_price'], y_pred_final)\n",
    "    scores.append(score)\n",
    "\n",
    "    print('(Fold %d) score, mse, mae, mape: %.5f, %.5f, %.5f, %.5f.' % (i, score, mse, mae, mape))\n",
    "    predicts_val.append(pd.DataFrame({'building_id': df_val['building_id'],\n",
    "                                      'total_price': df_val['total_price'],\n",
    "                                      'total_price_predict': y_pred_final}))\n",
    "\n",
    "    # Predict on test set\n",
    "    y_pred_test = model.predict(df_test[cols_feats])\n",
    "    predicts_test['target_{}'.format(i)] = y_pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total run time: 4502.034106731415\n"
     ]
    }
   ],
   "source": [
    "print('Total run time:', time.time() - t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kfold cv score (mean, stdev): 5870.873059133417 40.648820945027694\n"
     ]
    }
   ],
   "source": [
    "print('Kfold cv score (mean, stdev):', np.mean(scores), np.std(scores)) # 5868.87314"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>target_0</th>\n",
       "      <th>target_1</th>\n",
       "      <th>target_2</th>\n",
       "      <th>target_3</th>\n",
       "      <th>target_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>15.156375</td>\n",
       "      <td>15.208815</td>\n",
       "      <td>15.193924</td>\n",
       "      <td>15.206551</td>\n",
       "      <td>15.123375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>13.148812</td>\n",
       "      <td>13.135953</td>\n",
       "      <td>13.139107</td>\n",
       "      <td>13.141314</td>\n",
       "      <td>13.141162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>13.652254</td>\n",
       "      <td>13.701793</td>\n",
       "      <td>13.737303</td>\n",
       "      <td>13.687300</td>\n",
       "      <td>13.695594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>14.822818</td>\n",
       "      <td>14.795493</td>\n",
       "      <td>14.779526</td>\n",
       "      <td>14.823116</td>\n",
       "      <td>14.761048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>12.116399</td>\n",
       "      <td>12.130549</td>\n",
       "      <td>12.062083</td>\n",
       "      <td>12.128068</td>\n",
       "      <td>12.142269</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id   target_0   target_1   target_2   target_3   target_4\n",
       "0  X5gsdTWGS3W7JJQB  15.156375  15.208815  15.193924  15.206551  15.123375\n",
       "1  BTshNOJyKHnT2YIT  13.148812  13.135953  13.139107  13.141314  13.141162\n",
       "2  dhdymr0lV8N5kZOT  13.652254  13.701793  13.737303  13.687300  13.695594\n",
       "3  VEwyGGMcD56w5BOc  14.822818  14.795493  14.779526  14.823116  14.761048\n",
       "4  wmUeMoJZfsqaSX9b  12.116399  12.130549  12.062083  12.128068  12.142269"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicts_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat(predicts_val, axis=0).to_csv('output/'+model_name+'-cv.csv', index=False)\n",
    "\n",
    "predicts_test['target_mean'] = \\\n",
    "    predicts_test[['target_'+str(i) for i in range(len(id_cv))]].mean(axis=1)\n",
    "predicts_test['total_price'] = post_processing(predicts_test['target_mean'].values, df_test)\n",
    "predicts_test[['building_id', 'total_price']].to_csv('output/'+model_name+'-test-kfold.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best iterations: [31888, 30298, 30226, 30921, 30332]\n",
      "Mean best iterations: 30733\n"
     ]
    }
   ],
   "source": [
    "print('Best iterations:', best_iterations)\n",
    "print('Mean best iterations:', int(np.mean(best_iterations)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One-fold prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'num_boost_round': 35342, 'verbose_eval': 2000}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb_other_params_one = {'num_boost_round': int(np.mean(best_iterations)*1.15),\n",
    "                        'verbose_eval': 2000,}\n",
    "lgb_other_params_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'task': 'train',\n",
       " 'boosting_type': 'gbdt',\n",
       " 'objective': 'mse',\n",
       " 'metric': 'mape',\n",
       " 'learning_rate': 0.01,\n",
       " 'max_depth': 16,\n",
       " 'num_leaves': 300,\n",
       " 'feature_fraction': 0.5,\n",
       " 'min_data_in_leaf': 10,\n",
       " 'lambda_l1': 0.01,\n",
       " 'lambda_l2': 0.1}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2000]\ttraining's mape: 0.00404721\n",
      "[4000]\ttraining's mape: 0.00233128\n",
      "[6000]\ttraining's mape: 0.0014915\n",
      "[8000]\ttraining's mape: 0.00100254\n",
      "[10000]\ttraining's mape: 0.000698549\n",
      "[12000]\ttraining's mape: 0.000498139\n",
      "[14000]\ttraining's mape: 0.000366147\n",
      "[16000]\ttraining's mape: 0.000273677\n",
      "[18000]\ttraining's mape: 0.000209747\n",
      "[20000]\ttraining's mape: 0.000163504\n",
      "[22000]\ttraining's mape: 0.000130322\n",
      "[24000]\ttraining's mape: 0.000105536\n",
      "[26000]\ttraining's mape: 8.74056e-05\n",
      "[28000]\ttraining's mape: 7.33881e-05\n",
      "[30000]\ttraining's mape: 6.26953e-05\n",
      "[32000]\ttraining's mape: 5.42605e-05\n",
      "[34000]\ttraining's mape: 4.77339e-05\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.526912e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.924241e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.096127e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>6.155550e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.062995e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>EtBjGAHmHCe9t7TZ</td>\n",
       "      <td>2.614249e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>hPNH34vmaZtvBtqc</td>\n",
       "      <td>1.210946e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>wXjeI38bYDMJJwZC</td>\n",
       "      <td>6.185917e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>fxZSGX6aPAFKU8W4</td>\n",
       "      <td>1.677557e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ewr0Fx6ign87OwaV</td>\n",
       "      <td>4.148779e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>gHKurnEP4AowzsLg</td>\n",
       "      <td>8.936285e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>PmLfTgY2FElLrTl0</td>\n",
       "      <td>5.451857e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>eM2NppIOwzW0o8iy</td>\n",
       "      <td>9.505499e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>dxxwNun97NH4WTrZ</td>\n",
       "      <td>3.180282e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>jykBfhh3vdeFUi3H</td>\n",
       "      <td>3.919766e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>NlXbvdFfmJZf3L18</td>\n",
       "      <td>2.196297e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>D7jaFWHCzSqLBwdt</td>\n",
       "      <td>9.870530e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>L10dBBdqGmemweSl</td>\n",
       "      <td>6.283330e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>OgB0AdiPKlElakKN</td>\n",
       "      <td>1.096357e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>StiWNN1GQrpPBOYt</td>\n",
       "      <td>2.588744e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         building_id   total_price\n",
       "0   X5gsdTWGS3W7JJQB  1.526912e+07\n",
       "1   BTshNOJyKHnT2YIT  3.924241e+06\n",
       "2   dhdymr0lV8N5kZOT  1.096127e+07\n",
       "3   VEwyGGMcD56w5BOc  6.155550e+06\n",
       "4   wmUeMoJZfsqaSX9b  1.062995e+06\n",
       "5   EtBjGAHmHCe9t7TZ  2.614249e+06\n",
       "6   hPNH34vmaZtvBtqc  1.210946e+07\n",
       "7   wXjeI38bYDMJJwZC  6.185917e+06\n",
       "8   fxZSGX6aPAFKU8W4  1.677557e+06\n",
       "9   ewr0Fx6ign87OwaV  4.148779e+06\n",
       "10  gHKurnEP4AowzsLg  8.936285e+05\n",
       "11  PmLfTgY2FElLrTl0  5.451857e+06\n",
       "12  eM2NppIOwzW0o8iy  9.505499e+06\n",
       "13  dxxwNun97NH4WTrZ  3.180282e+06\n",
       "14  jykBfhh3vdeFUi3H  3.919766e+06\n",
       "15  NlXbvdFfmJZf3L18  2.196297e+07\n",
       "16  D7jaFWHCzSqLBwdt  9.870530e+05\n",
       "17  L10dBBdqGmemweSl  6.283330e+06\n",
       "18  OgB0AdiPKlElakKN  1.096357e+06\n",
       "19  StiWNN1GQrpPBOYt  2.588744e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_train = train.copy()\n",
    "df_test = test.copy()\n",
    "\n",
    "# feat eng\n",
    "feat_eng = FeatureEngineering(cols_num, cols_cat, cols_feats, col_target)\n",
    "df_train = feat_eng.fit_transform(df_train)\n",
    "df_test = feat_eng.transform(df_test)\n",
    "\n",
    "# Construct lgb dataset\n",
    "lgb_train = lgb.Dataset(df_train[cols_feats], df_train[col_target]).construct()\n",
    "\n",
    "# train model    \n",
    "model = lgb.train(params, lgb_train, valid_sets=lgb_train, **lgb_other_params_one)\n",
    "\n",
    "y_pred = model.predict(df_test[cols_feats])\n",
    "y_pred_final = post_processing(y_pred, df_test)\n",
    "\n",
    "df_test_pred = pd.DataFrame({'building_id': df_test['building_id'], \n",
    "                             'total_price': y_pred_final})\n",
    "display(df_test_pred.head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_pred.to_csv('output/'+model_name+'-test-one.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb.plot_importance(model, figsize=(12, 30), height=0.5) # 050, 051, 052\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb.plot_importance(model, figsize=(12, 30), height=0.5, importance_type='gain') # 050, 051, 052\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "193.785px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}