model-09-predict-lgb-rm-75-no_hist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "remove features based on feature importance results (exp-l04-per-area)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pickle\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from utilities import cal_mape\n",
    "\n",
    "from utilities import cal_score, convert_types, create_dt_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'model-09-lgb-feats-selection-75'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_log_target = True\n",
    "is_per_area_target = True\n",
    "\n",
    "map_target = {\n",
    "    (True, True):   'log_per_area_total_price',\n",
    "    (True, False):  'log_total_price',\n",
    "    (False, True):  'per_area_total_price',\n",
    "    (False, False): 'total_price'\n",
    "}\n",
    "col_target = map_target[(is_log_target, is_per_area_target)]\n",
    "\n",
    "#cols_feats_add = ['encoded_latlon', 'building_per_land_area'] # add feats into model\n",
    "cols_num_add = ['building_per_land_area']\n",
    "cols_cat_add = ['encoded_latlon']\n",
    "\n",
    "#cols_feats_rm = [] # TODO: implement before using it\n",
    "#['day_in_year_txn_dt', 'day_in_year_building_complete_dt', 'dt_diff_building_complete_dt_txn_dt'] # dt num feats\n",
    "#['dow_txn_dt', 'dow_building_complete_dt', 'month_txn_dt', 'month_building_complete_dt',\n",
    "# 'quarter_txn_dt', 'quarter_building_complete_dt', 'year_txn_dt', 'year_building_complete_dt'] # dt cat feats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_processing(df, cols_num, cols_cat): #, cols_feats\n",
    "    # Convert types\n",
    "    df[cols_num] = df[cols_num].astype('float32')\n",
    "    \n",
    "    # generate target\n",
    "    if 'total_price' in df:\n",
    "        df['total_price'] = df['total_price'].astype('float32')\n",
    "        df['log_total_price'] = np.log1p(df['total_price'])\n",
    "        df['per_area_total_price'] = df['total_price'] / df['building_area']\n",
    "        df['log_per_area_total_price'] = np.log1p(df['per_area_total_price'])\n",
    "        df['target'] = df[col_target]\n",
    "\n",
    "    # generate lat/lon encoded\n",
    "    kmeans = pickle.load(open('./object_kmeans_k6_combined.pkl', 'rb'))\n",
    "    df['encoded_latlon'] = kmeans.predict(df[['lat','lon']])\n",
    "    \n",
    "    # generate dt feats\n",
    "    df = create_dt_feats(df, 'txn_dt')\n",
    "    df = create_dt_feats(df, 'building_complete_dt')\n",
    "    df['dt_diff_building_complete_dt_txn_dt'] = df['txn_dt'] - df['building_complete_dt']\n",
    "    \n",
    "    # generate building to land ratio\n",
    "    df['building_per_land_area'] = df['building_area'] / df['land_area']\n",
    " \n",
    "    #cols_feats = cols_num + cols_cat + cols_feats_add\n",
    "    \n",
    "    # fix town and city\n",
    "#    df['town'] = df['city'].astype('str')+'-'+df['town'].astype('str')\n",
    "#    df['village'] = df['town'].astype('str')+'-'+df['village'].astype('str')\n",
    "    \n",
    "    return df  #, cols_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from feature_engineering import CategoricalColumnsEncoder\n",
    "from feature_engineering import TargetMeanEncoding\n",
    "from feature_engineering import MultiLabelEncoding\n",
    "\n",
    "class FeatureEngineering():\n",
    "    def __init__(self, cols_num, cols_cat, cols_feats, col_target, n_ran_encode=4):\n",
    "        # cols\n",
    "        self.cols_num = cols_num\n",
    "        self.cols_cat = cols_cat\n",
    "        self.cols_feats = cols_feats\n",
    "        self.col_target = col_target\n",
    "        \n",
    "        # Numbers\n",
    "        self.n_ran_encode = n_ran_encode\n",
    "        \n",
    "        # encoders\n",
    "        #self.tar_encoder = TargetMeanEncoding(col_target = self.col_target)\n",
    "        #self.ran_encoder = MultiLabelEncoding(self.n_ran_encode)\n",
    "        self.cat_astyper = CategoricalColumnsEncoder(mode='pandas')\n",
    "        \n",
    "    def fit_transform(self, df):\n",
    "        df = df.copy()\n",
    "    \n",
    "#        encoded1 = self.tar_encoder.fit_transform(df[self.cols_cat], df[self.col_target])\n",
    "#        encoded2 = self.ran_encoder.fit_transform(df[self.cols_cat])\n",
    "        self.cat_astyper.fit_transform(df, self.cols_cat)    \n",
    "        return df\n",
    "#        return pd.concat([df, encoded1, encoded2], axis=1)\n",
    "    \n",
    "    def transform(self, df):\n",
    "        df = df.copy()\n",
    "#        encoded1 = self.tar_encoder.transform(df[self.cols_cat])\n",
    "#        encoded2 = self.ran_encoder.transform(df[self.cols_cat])     # TODO: add generated feats into cols_feats\n",
    "        self.cat_astyper.transform(df)\n",
    "        return df\n",
    "#        return pd.concat([df, encoded1, encoded2], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_processing(y_pred, df):\n",
    "    \"\"\"Args: \n",
    "          y_pred (1d np.array): raw prediction y\n",
    "          df (pd.Dataframe): input feature df\n",
    "       Return: np.array\n",
    "    \"\"\"\n",
    "    y_pred_final = y_pred.copy()\n",
    "    \n",
    "    if is_log_target:\n",
    "        y_pred_final = np.expm1(y_pred_final)\n",
    "        y_pred_final = np.clip(y_pred_final, 0, None)\n",
    "\n",
    "    if is_per_area_target:\n",
    "        y_pred_final = y_pred_final * df['building_area'].values\n",
    "        \n",
    "    return y_pred_final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_num = ['txn_dt', 'total_floor', 'building_complete_dt', 'parking_area', 'parking_price', 'txn_floor',\n",
    "            'land_area', 'building_area', 'lat', 'lon', 'village_income_median', 'town_population', 'town_area',\n",
    "            'town_population_density', 'doc_rate', 'master_rate', 'bachelor_rate', 'jobschool_rate',\n",
    "            'highschool_rate', 'junior_rate', 'elementary_rate', 'born_rate', 'death_rate', 'marriage_rate',\n",
    "            'divorce_rate', 'N_50', 'N_500', 'N_1000', 'N_5000', 'N_10000', 'I_10', 'I_50', 'I_100', 'I_250',\n",
    "            'I_500', 'I_1000', 'I_5000', 'I_10000', 'I_MIN', 'II_10', 'II_50', 'II_100', 'II_250', 'II_500',\n",
    "            'II_1000', 'II_5000', 'II_10000', 'II_MIN', 'III_10', 'III_50', 'III_100', 'III_250', 'III_500',\n",
    "            'III_1000', 'III_5000', 'III_10000', 'III_MIN', 'IV_10', 'IV_50', 'IV_100', 'IV_250', 'IV_500',\n",
    "            'IV_1000', 'IV_5000', 'IV_10000', 'IV_MIN', 'V_10', 'V_50', 'V_100', 'V_250', 'V_500', 'V_1000',\n",
    "            'V_5000', 'V_10000', 'V_MIN', 'VI_50', 'VI_100', 'VI_250', 'VI_500', 'VI_1000', 'VI_5000', 'VI_10000',\n",
    "            'VI_MIN', 'VII_10', 'VII_50', 'VII_100', 'VII_250', 'VII_500', 'VII_1000', 'VII_5000', 'VII_10000',\n",
    "            'VII_MIN', 'VIII_10', 'VIII_50', 'VIII_100', 'VIII_250', 'VIII_500', 'VIII_1000', 'VIII_5000',\n",
    "            'VIII_10000', 'VIII_MIN', 'IX_10', 'IX_50', 'IX_100', 'IX_250', 'IX_500', 'IX_1000', 'IX_5000',\n",
    "            'IX_10000', 'IX_MIN', 'X_10', 'X_50', 'X_100', 'X_250', 'X_500', 'X_1000', 'X_5000', 'X_10000', 'X_MIN',\n",
    "            'XI_10', 'XI_50', 'XI_100', 'XI_250', 'XI_500', 'XI_1000', 'XI_5000', 'XI_10000', 'XI_MIN', 'XII_10',\n",
    "            'XII_50', 'XII_100', 'XII_250', 'XII_500', 'XII_1000', 'XII_5000', 'XII_10000', 'XII_MIN', 'XIII_10',\n",
    "            'XIII_50', 'XIII_100', 'XIII_250', 'XIII_500', 'XIII_1000', 'XIII_5000', 'XIII_10000', 'XIII_MIN',\n",
    "            'XIV_10', 'XIV_50', 'XIV_100', 'XIV_250', 'XIV_500', 'XIV_1000', 'XIV_5000', 'XIV_10000', 'XIV_MIN']\n",
    "cols_cat = ['building_material', 'city', 'building_type', 'building_use', 'parking_way', 'town', 'village',\n",
    "            'I_index_50', 'I_index_500', 'I_index_1000', 'II_index_50', 'II_index_500', 'II_index_1000',\n",
    "            'III_index_50', 'III_index_500', 'III_index_1000', 'IV_index_50', 'IV_index_500', 'IV_index_1000',\n",
    "            'IV_index_5000', 'V_index_50', 'V_index_500', 'V_index_1000', 'VI_10', 'VI_index_50', 'VI_index_500',\n",
    "            'VI_index_1000', 'VII_index_50', 'VII_index_500', 'VII_index_1000', 'VIII_index_50', 'VIII_index_500',\n",
    "            'VIII_index_1000', 'IX_index_50', 'IX_index_500', 'IX_index_1000', 'IX_index_5000', 'X_index_50',\n",
    "            'X_index_500', 'X_index_1000', 'XI_index_50', 'XI_index_500', 'XI_index_1000', 'XI_index_5000',\n",
    "            'XI_index_10000', 'XII_index_50', 'XII_index_500', 'XII_index_1000', 'XIII_index_50', 'XIII_index_500',\n",
    "            'XIII_index_1000', 'XIII_index_5000', 'XIII_index_10000', 'XIV_index_50', 'XIV_index_500',\n",
    "            'XIV_index_1000']\n",
    "cols_feats = cols_num + cols_cat\n",
    "\n",
    "# informational cols\n",
    "cols_dt = ['txn_dt', 'building_complete_dt']\n",
    "cols_bin = ['I_index_50', 'I_index_500', 'I_index_1000', 'II_index_50', 'II_index_500', 'II_index_1000',\n",
    "            'III_index_50', 'III_index_500', 'III_index_1000', 'IV_index_50', 'IV_index_500', 'IV_index_1000',\n",
    "            'IV_index_5000', 'V_index_50', 'V_index_500', 'V_index_1000', 'VI_10', 'VI_index_50', 'VI_index_500',\n",
    "            'VI_index_1000', 'VII_index_50', 'VII_index_500', 'VII_index_1000', 'VIII_index_50', 'VIII_index_500',\n",
    "            'VIII_index_1000', 'IX_index_50', 'IX_index_500', 'IX_index_1000', 'IX_index_5000', 'X_index_50',\n",
    "            'X_index_500', 'X_index_1000', 'XI_index_50', 'XI_index_500', 'XI_index_1000', 'XI_index_5000',\n",
    "            'XI_index_10000', 'XII_index_50', 'XII_index_500', 'XII_index_1000', 'XIII_index_50', 'XIII_index_500',\n",
    "            'XIII_index_1000', 'XIII_index_5000', 'XIII_index_10000', 'XIV_index_50', 'XIV_index_500',\n",
    "            'XIV_index_1000']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('dataset/train.csv', dtype=object)\n",
    "test = pd.read_csv('dataset/test.csv', dtype=object)\n",
    "train.head(5)\n",
    "test.head(5)\n",
    "\n",
    "id_cv = []\n",
    "for i in range(5):\n",
    "    id_one_fold = pd.read_csv(f'dataset/id_cv{i}.csv', dtype=object)['building_id']\n",
    "    id_cv.append(id_one_fold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python36\\lib\\site-packages\\sklearn\\base.py:306: UserWarning: Trying to unpickle estimator KMeans from version 0.20.3 when using version 0.21.2. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "c:\\python36\\lib\\site-packages\\sklearn\\base.py:306: UserWarning: Trying to unpickle estimator KMeans from version 0.20.3 when using version 0.21.2. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "train = pre_processing(train, cols_num, cols_cat)\n",
    "test = pre_processing(test, cols_num, cols_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_num = cols_num + [c for c in cols_num_add if c not in cols_num]\n",
    "cols_cat = cols_cat + [c for c in cols_cat_add if c not in cols_cat]\n",
    "cols_feats = cols_num + cols_cat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove feats based on shuffle results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 'divorce_rate' 5709.20035 3.333296666667593]\n",
      " [1 'I_250' 5709.200316666666 3.333263333333889]\n",
      " [2 'XI_100' 5708.200346666667 2.33329333333495]\n",
      " [3 'VI_index_500' 5707.867050000001 1.9999966666682667]\n",
      " [4 'IX_index_50' 5707.86704 1.9999866666676098]\n",
      " [5 'VI_500' 5707.866983333333 1.9999300000008589]\n",
      " [6 'XIV_10' 5707.533710000001 1.666656666668132]\n",
      " [7 'XI_index_500' 5707.200386666666 1.3333333333339397]\n",
      " [8 'VI_index_1000' 5707.200383333334 1.3333300000012969]\n",
      " [9 'V_100' 5707.200373333333 1.33332000000064]\n",
      " [10 'VII_10' 5707.200363333334 1.333310000001802]\n",
      " [11 'X_100' 5707.200336666666 1.3332833333333838]\n",
      " [12 'XII_index_50' 5706.867056666667 1.0000033333344618]\n",
      " [15 'X_index_500' 5706.867050000001 0.9999966666682667]\n",
      " [14 'X_10' 5706.867050000001 0.9999966666682667]\n",
      " [13 'II_10' 5706.867050000001 0.9999966666682667]\n",
      " [16 'II_index_500' 5706.867046666666 0.9999933333338049]\n",
      " [17 'V_index_1000' 5706.867046666666 0.9999933333338049]\n",
      " [18 'XIII_10' 5706.867043333335 0.9999900000020716]\n",
      " [19 'XIV_index_50' 5706.867043333335 0.9999900000020716]\n",
      " [20 'VIII_index_500' 5706.86704 0.9999866666676098]\n",
      " [21 'IX_50' 5706.867016666667 0.9999633333345628]\n",
      " [22 'XI_index_1000' 5706.533726666666 0.6666733333331649]\n",
      " [23 'XI_10' 5706.533719999999 0.6666666666669698]\n",
      " [24 'II_index_1000' 5706.533719999999 0.6666666666669698]\n",
      " [25 'VII_index_500' 5706.533719999999 0.6666666666669698]\n",
      " [26 'X_index_1000' 5706.533719999999 0.6666666666669698]\n",
      " [30 'XIII_index_1000' 5706.533716666668 0.6666633333352365]\n",
      " [31 'XIV_index_500' 5706.533716666668 0.6666633333352365]\n",
      " [27 'VI_100' 5706.533716666668 0.6666633333352365]\n",
      " [29 'IX_index_500' 5706.533716666668 0.6666633333352365]\n",
      " [28 'I_index_1000' 5706.533716666668 0.6666633333352365]\n",
      " [32 'XIII_index_500' 5706.533713333333 0.6666600000007747]\n",
      " [33 'V_10' 5706.533706666666 0.6666533333336702]\n",
      " [34 'VII_index_50' 5706.533706666666 0.6666533333336702]\n",
      " [35 'I_500' 5706.533573333333 0.6665200000006735]\n",
      " [36 'IV_index_50' 5706.200386666666 0.33333333333393966]\n",
      " [37 'V_index_500' 5706.200386666666 0.33333333333393966]\n",
      " [38 'VI_index_50' 5706.200386666666 0.33333333333393966]\n",
      " [39 'XIII_index_5000' 5706.200386666666 0.33333333333393966]\n",
      " [40 'VI_50' 5706.200383333334 0.33333000000129687]\n",
      " [41 'III_index_1000' 5706.200383333334 0.33333000000129687]\n",
      " [44 'I_index_50' 5706.200379999999 0.3333266666668351]\n",
      " [42 'I_50' 5706.200379999999 0.3333266666668351]\n",
      " [43 'IV_50' 5706.200379999999 0.3333266666668351]\n",
      " [45 'III_index_50' 5706.200370000001 0.3333166666689067]\n",
      " [46 'VI_250' 5706.200366666667 0.3333133333344449]\n",
      " [47 'XI_index_50' 5706.200366666667 0.3333133333344449]\n",
      " [48 'VI_1000' 5706.200263333333 0.33321000000069034]\n",
      " [49 'XII_index_500' 5705.867056666667 3.333334461785853e-06]\n",
      " [59 'XIV_index_1000' 5705.8670533333325 0.0]\n",
      " [58 'XIII_index_50' 5705.8670533333325 0.0]\n",
      " [57 'XII_index_1000' 5705.8670533333325 0.0]\n",
      " [55 'IX_index_5000' 5705.8670533333325 0.0]\n",
      " [56 'XI_index_10000' 5705.8670533333325 0.0]\n",
      " [53 'VI_10' 5705.8670533333325 0.0]\n",
      " [54 'VII_index_1000' 5705.8670533333325 0.0]\n",
      " [52 'IV_index_5000' 5705.8670533333325 0.0]\n",
      " [51 'I_10' 5705.8670533333325 0.0]\n",
      " [50 'none' 5705.8670533333325 0.0]\n",
      " [60 'VIII_index_1000' 5705.867050000001 -3.3333317333017476e-06]\n",
      " [61 'XI_index_5000' 5705.867050000001 -3.3333317333017476e-06]\n",
      " [62 'I_index_500' 5705.86704 -1.3333332390175201e-05]\n",
      " [63 'II_index_50' 5705.86704 -1.3333332390175201e-05]\n",
      " [64 'IX_index_1000' 5705.867036666667 -1.666666503297165e-05]\n",
      " [65 'I_100' 5705.86703 -2.3333332137553953e-05]\n",
      " [66 'II_50' 5705.8670133333335 -3.9999998989515007e-05]\n",
      " [67 'N_500' 5705.866976666667 -7.666666533623356e-05]\n",
      " [68 'N_5000' 5705.533719999999 -0.33333333333303017]\n",
      " [69 'IX_10' 5705.533713333333 -0.33333999999922526]\n",
      " [70 'IV_index_1000' 5705.533713333333 -0.33333999999922526]\n",
      " [71 'X_index_50' 5705.533713333333 -0.33333999999922526]\n",
      " [72 'IV_10' 5705.533710000001 -0.33334333333186805]\n",
      " [73 'XIII_index_10000' 5705.533710000001 -0.33334333333186805]\n",
      " [74 'XII_10' 5705.533630000001 -0.33342333333166607]\n",
      " [75 'III_10' 5705.200383333334 -0.6666699999987031]\n",
      " [76 'X_50' 5705.200370000001 -0.6666833333310933]\n",
      " [77 'IV_index_500' 5705.200370000001 -0.6666833333310933]\n",
      " [78 'VIII_index_50' 5705.200363333334 -0.6666899999981979]\n",
      " [79 'XI_50' 5705.20035 -0.666703333332407]\n",
      " [80 'N_10000' 5705.200326666666 -0.6667266666663636]\n",
      " [81 'V_index_50' 5704.867036666667 -1.000016666665033]\n",
      " [82 'X_500' 5704.86689 -1.0001633333322388]\n",
      " [83 'X_250' 5704.866873333333 -1.0001799999990908]\n",
      " [84 'III_index_500' 5704.533719999999 -1.3333333333330302]\n",
      " [85 'XIII_50' 5704.533710000001 -1.333343333331868]\n",
      " [86 'XIII_100' 5704.533710000001 -1.333343333331868]\n",
      " [87 'V_50' 5704.200356666667 -1.6666966666653025]\n",
      " [88 'IX_100' 5704.2002600000005 -1.6667933333319525]\n",
      " [89 'XIV_250' 5704.200026666666 -1.667026666666061]\n",
      " [90 'IV_100' 5703.867026666667 -2.00002666666569]\n",
      " [91 'IV_250' 5703.866953333333 -2.0000999999992928]\n",
      " [92 'III_100' 5703.533653333333 -2.3333999999995285]\n",
      " [93 'XIV_100' 5703.533503333333 -2.333549999999377]\n",
      " [94 'XIII_250' 5703.200316666666 -2.666736666666111]\n",
      " [95 'XI_250' 5703.200193333333 -2.6668599999993603]\n",
      " [96 'VIII_10' 5702.866996666667 -3.0000566666658415]\n",
      " [97 'IX_250' 5702.200013333334 -3.667039999998451]\n",
      " [98 'IV_500' 5701.866936666666 -4.000116666666145]\n",
      " [99 'IX_5000' 5701.533173333333 -4.333879999999226]\n",
      " [100 'XII_100' 5701.2001666666665 -4.66688666666596]\n",
      " [101 'II_100' 5700.866966666667 -5.000086666665084]\n",
      " [102 'N_1000' 5700.866936666666 -5.000116666666145]\n",
      " [103 'born_rate' 5700.86682 -5.0002333333322895]\n",
      " [104 'junior_rate' 5700.5335000000005 -5.33355333333202]\n",
      " [105 'VIII_50' 5700.533383333334 -5.333669999998165]\n",
      " [106 'marriage_rate' 5700.533316666667 -5.333736666665573]\n",
      " [107 'XIV_50' 5699.533613333333 -6.3334399999994275]\n",
      " [108 'VII_50' 5699.533456666667 -6.333596666665471]\n",
      " [109 'IV_1000' 5699.199853333333 -6.667199999999866]\n",
      " [110 'II_250' 5698.53337 -7.333683333332374]\n",
      " [111 'XIII_500' 5698.200293333332 -7.6667600000000675]\n",
      " [112 'III_50' 5698.200069999999 -7.666983333333519]\n",
      " [113 'II_10000' 5698.199886666666 -7.667166666666162]\n",
      " [114 'IV_5000' 5697.866536666667 -8.000516666665135]\n",
      " [115 'XIII_1000' 5697.533463333333 -8.333589999999276]\n",
      " [116 'III_250' 5697.200150000001 -8.666903333331902]\n",
      " [117 'VIII_100' 5696.866476666667 -9.000576666665438]\n",
      " [118 'XII_50' 5696.200313333335 -9.666739999997844]\n",
      " [119 'N_50' 5696.19989 -9.66716333333261]\n",
      " [120 'VIII_5000' 5695.533113333334 -10.33393999999862]\n",
      " [121 'I_1000' 5694.866853333333 -11.000199999999495]\n",
      " [122 'VI_MIN' 5694.866663333334 -11.000389999998333]\n",
      " [123 'XIV_500' 5694.866446666667 -11.00060666666559]\n",
      " [124 'VII_100' 5694.53343 -11.33362333333207]\n",
      " [125 'XI_500' 5693.8668800000005 -12.000173333331986]\n",
      " [126 'IV_10000' 5693.532886666667 -12.334166666665624]\n",
      " [127 'V_MIN' 5692.86606 -13.00099333333219]\n",
      " [128 'death_rate' 5692.5331633333335 -13.333889999998974]\n",
      " [129 'VIII_10000' 5692.199853333333 -13.667199999999866]\n",
      " [130 'XIII_MIN' 5691.533060000001 -14.333993333331819]\n",
      " [131 'V_500' 5690.199906666668 -15.667146666664848]\n",
      " [132 'I_MIN' 5690.19978 -15.66727333333256]\n",
      " [133 'V_250' 5689.200093333333 -16.666959999999563]\n",
      " [134 'IV_MIN' 5689.1995566666665 -16.66749666666601]\n",
      " [135 'III_500' 5688.866743333333 -17.000309999999445]\n",
      " [136 'IX_500' 5688.86639 -17.00066333333234]\n",
      " [137 'XI_5000' 5688.533226666666 -17.333826666666027]\n",
      " [138 'XII_250' 5688.533106666667 -17.333946666665724]\n",
      " [139 'III_1000' 5687.866279999999 -18.0007733333332]\n",
      " [140 'XI_MIN' 5687.532873333334 -18.334179999998923]\n",
      " [141 'X_10000' 5687.532800000001 -18.334253333331617]\n",
      " [142 'II_500' 5686.532996666666 -19.334056666666584]\n",
      " [143 'III_5000' 5684.866556666667 -21.00049666666564]\n",
      " [144 'V_1000' 5683.866256666667 -22.000796666665337]\n",
      " [145 'III_10000' 5683.5324900000005 -22.33456333333197]\n",
      " [146 'VII_MIN' 5683.199209999999 -22.66784333333362]\n",
      " [147 'XII_500' 5683.199173333334 -22.66787999999815]\n",
      " [148 'III_MIN' 5682.532556666666 -23.334496666666382]\n",
      " [149 'VIII_250' 5681.86623 -24.000823333332846]\n",
      " [150 'XIV_1000' 5680.866440000001 -25.000613333331785]\n",
      " [151 'town_area' 5680.533066666667 -25.333986666665623]\n",
      " [152 'XIV_MIN' 5680.19969 -25.667363333332105]\n",
      " [153 'II_MIN' 5677.53274 -28.33431333333283]\n",
      " [154 'XII_5000' 5676.8655733333335 -29.001479999998992]\n",
      " [155 'building_material' 5676.865563333333 -29.00148999999965]\n",
      " [156 'X_1000' 5675.865993333333 -30.001059999999597]\n",
      " [157 'I_5000' 5675.532426666668 -30.334626666664917]\n",
      " [158 'XI_10000' 5675.199393333333 -30.66765999999916]\n",
      " [159 'XII_10000' 5674.865669999999 -31.00138333333325]\n",
      " [160 'elementary_rate' 5674.532066666667 -31.334986666665827]\n",
      " [161 'town_population_density' 5674.199496666667 -31.667556666665405]\n",
      " [162 'IX_MIN' 5673.5324900000005 -32.33456333333197]\n",
      " [163 'XII_MIN' 5672.199463333333 -33.66758999999911]\n",
      " [164 'X_MIN' 5672.199316666666 -33.667736666666315]\n",
      " [165 'VII_250' 5671.865873333333 -34.001179999999295]\n",
      " [166 'XIV_10000' 5669.865906666667 -36.00114666666559]\n",
      " [167 'X_5000' 5669.865403333333 -36.001649999999245]\n",
      " [168 'parking_area' 5665.866163333333 -40.000889999999345]\n",
      " [169 'IX_10000' 5665.53269 -40.334363333332476]\n",
      " [170 'highschool_rate' 5664.532736666667 -41.33431666666547]\n",
      " [171 'II_5000' 5662.532619999999 -43.334433333333436]\n",
      " [172 'V_5000' 5661.19843 -44.66862333333211]\n",
      " [173 'VIII_MIN' 5660.532049999999 -45.33500333333359]\n",
      " [174 'VIII_500' 5658.531816666667 -47.33523666666588]\n",
      " [175 'VIII_1000' 5657.86419 -48.00286333333224]\n",
      " [176 'I_10000' 5655.531346666667 -50.33570666666583]\n",
      " [177 'building_use' 5652.19898 -53.66807333333236]\n",
      " [178 'XII_1000' 5644.19792 -61.669133333332866]\n",
      " [179 'VI_5000' 5641.86575 -64.00130333333254]\n",
      " [180 'lat' 5637.531133333333 -68.33591999999953]\n",
      " [181 'VII_5000' 5631.530966666666 -74.33608666666623]\n",
      " [182 'VII_500' 5629.531536666666 -76.33551666666608]\n",
      " [183 'town_population' 5626.198586666666 -79.66846666666606]\n",
      " [184 'city' 5624.864426666667 -81.00262666666549]\n",
      " [185 'II_1000' 5618.53164 -87.33541333333233]\n",
      " [186 'IX_1000' 5609.530946666667 -96.33610666666573]\n",
      " [187 'XI_1000' 5600.864693333333 -105.0023599999995]\n",
      " [188 'jobschool_rate' 5577.52952 -128.33753333333243]\n",
      " [189 'bachelor_rate' 5564.530006666667 -141.33704666666563]\n",
      " [190 'XIV_5000' 5544.529993333334 -161.33705999999893]\n",
      " [191 'V_10000' 5527.5279433333335 -178.33910999999898]\n",
      " [192 'VII_1000' 5483.195199999999 -222.67185333333327]\n",
      " [193 'parking_way' 5481.528866666667 -224.33818666666502]\n",
      " [194 'XIII_5000' 5403.193223333333 -302.6738299999997]\n",
      " [195 'lon' 5361.52429 -344.3427633333322]\n",
      " [196 'parking_price' 5360.190783333333 -345.67626999999993]\n",
      " [197 'master_rate' 5210.856313333333 -495.0107399999997]\n",
      " [198 'building_type' 5193.523143333333 -512.3439099999996]\n",
      " [199 'village_income_median' 5009.84994 -696.0171133333324]\n",
      " [200 'total_floor' 4984.18339 -721.6836633333323]\n",
      " [201 'building_area' 4936.506333333334 -769.3607199999988]\n",
      " [202 'doc_rate' 4886.513366666666 -819.3536866666664]\n",
      " [203 'village' 4852.180716666666 -853.6863366666666]\n",
      " [204 'VII_10000' 4688.84273 -1017.0243233333322]\n",
      " [205 'land_area' 4551.80975 -1154.0573033333321]\n",
      " [206 'VI_10000' 4376.823850000001 -1329.0432033333318]\n",
      " [207 'txn_floor' 4149.4685 -1556.3985533333325]\n",
      " [208 'txn_dt' 3962.7975066666663 -1743.0695466666662]\n",
      " [209 'building_complete_dt' 3433.7606266666667 -2272.106426666666]\n",
      " [210 'XIII_10000' 2914.651653333333 -2791.2153999999996]\n",
      " [211 'town' 2516.730063333333 -3189.1369899999995]]\n",
      "75\n",
      "['divorce_rate', 'I_250', 'XI_100', 'VI_index_500', 'IX_index_50', 'VI_500', 'XIV_10', 'XI_index_500', 'VI_index_1000', 'V_100', 'VII_10', 'X_100', 'XII_index_50', 'X_index_500', 'X_10', 'II_10', 'II_index_500', 'V_index_1000', 'XIII_10', 'XIV_index_50', 'VIII_index_500', 'IX_50', 'XI_index_1000', 'XI_10', 'II_index_1000', 'VII_index_500', 'X_index_1000', 'XIII_index_1000', 'XIV_index_500', 'VI_100', 'IX_index_500', 'I_index_1000', 'XIII_index_500', 'V_10', 'VII_index_50', 'I_500', 'IV_index_50', 'V_index_500', 'VI_index_50', 'XIII_index_5000', 'VI_50', 'III_index_1000', 'I_index_50', 'I_50', 'IV_50', 'III_index_50', 'VI_250', 'XI_index_50', 'VI_1000', 'XII_index_500', 'XIV_index_1000', 'XIII_index_50', 'XII_index_1000', 'IX_index_5000', 'XI_index_10000', 'VI_10', 'VII_index_1000', 'IV_index_5000', 'I_10', 'VIII_index_1000', 'XI_index_5000', 'I_index_500', 'II_index_50', 'IX_index_1000', 'I_100', 'II_50', 'N_500', 'N_5000', 'IX_10', 'IV_index_1000', 'X_index_50', 'IV_10', 'XIII_index_10000', 'XII_10', 'III_10']\n",
      "N feats (before): 213\n",
      "N feats (after): 138\n",
      "['VIII_500', 'lat', 'VI_MIN', 'VIII_1000', 'XIV_250', 'building_use', 'VIII_index_50', 'X_500', 'V_10000', 'VIII_5000', 'IV_5000', 'building_material', 'IV_100', 'master_rate', 'XIII_500', 'II_5000', 'V_5000', 'village', 'IX_MIN', 'lon', 'II_MIN', 'IX_1000', 'XIV_500', 'highschool_rate', 'X_5000', 'XI_10000', 'parking_price', 'born_rate', 'V_MIN', 'land_area', 'VII_250', 'parking_way', 'X_250', 'XII_250', 'N_50', 'XIV_5000', 'XIV_MIN', 'III_index_500', 'VIII_250', 'city', 'III_500', 'I_MIN', 'IX_500', 'building_area', 'IX_10000', 'XII_100', 'X_50', 'II_250', 'X_10000', 'XII_5000', 'VII_100', 'XIII_MIN', 'III_250', 'IV_10000', 'building_complete_dt', 'elementary_rate', 'XII_1000', 'building_per_land_area', 'XIII_1000', 'town_population_density', 'X_1000', 'VI_10000', 'txn_floor', 'XI_5000', 'town_area', 'X_MIN', 'N_1000', 'XIV_50', 'III_MIN', 'II_500', 'town_population', 'XI_MIN', 'III_5000', 'V_50', 'II_1000', 'encoded_latlon', 'IX_100', 'XII_500', 'town', 'XI_250', 'XII_10000', 'IV_MIN', 'txn_dt', 'III_10000', 'VI_5000', 'village_income_median', 'III_1000', 'VIII_10', 'VII_500', 'VII_MIN', 'IV_index_500', 'V_250', 'IV_250', 'total_floor', 'XIV_1000', 'XI_500', 'III_50', 'I_1000', 'VIII_100', 'III_100', 'bachelor_rate', 'VIII_MIN', 'VII_10000', 'jobschool_rate', 'VIII_10000', 'XIII_100', 'XIII_50', 'doc_rate', 'XII_50', 'junior_rate', 'XIV_10000', 'VIII_50', 'VII_5000', 'XIV_100', 'XI_1000', 'IV_1000', 'XIII_250', 'V_index_50', 'I_10000', 'building_type', 'XI_50', 'II_100', 'IV_500', 'V_500', 'II_10000', 'marriage_rate', 'XIII_10000', 'N_10000', 'V_1000', 'IX_5000', 'VII_50', 'IX_250', 'parking_area', 'I_5000', 'XIII_5000', 'death_rate', 'VII_1000', 'XII_MIN']\n"
     ]
    }
   ],
   "source": [
    "# PARS #\n",
    "N_feats_rm = 75\n",
    "# PARS #\n",
    "\n",
    "from vars_03 import score_base_per_area, shuffle_results_per_area\n",
    "\n",
    "feat_imp = pd.DataFrame({'feats': shuffle_results_per_area[:,0],\n",
    "                         'avg':shuffle_results_per_area[:,1]})\n",
    "feat_imp['diff'] = feat_imp['avg'] - score_base_per_area\n",
    "feat_imp = feat_imp.sort_values(by='diff', ascending=False)\n",
    "print(feat_imp.reset_index().values)\n",
    "\n",
    "feats_rm = list(feat_imp.iloc[:(N_feats_rm+1)]['feats'].values)\n",
    "feats_rm.remove('none')\n",
    "print(len(feats_rm))\n",
    "print(feats_rm)\n",
    "\n",
    "print('N feats (before):', len(cols_feats))\n",
    "cols_feats = list(set(cols_feats) - set(feats_rm))\n",
    "print('N feats (after):', len(cols_feats))\n",
    "print(cols_feats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params\n",
    "params = {'task': 'train',\n",
    "          'boosting_type': 'gbdt',\n",
    "          'objective': 'mse',\n",
    "          'metric': 'mape',\n",
    "          'learning_rate': 0.01,\n",
    "          'max_depth': 24,\n",
    "          'num_leaves': 300,           # may reduce in dim-reduction exp\n",
    "          'feature_fraction': 0.5,\n",
    "          'min_data_in_leaf': 10,\n",
    "          'lambda_l1': 0.1,\n",
    "          'lambda_l2': 0.0,}\n",
    "\n",
    "lgb_other_params = {'num_boost_round': 100000,\n",
    "                    'verbose_eval': 2000,\n",
    "                    'early_stopping_rounds': 1000,}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### kfold cross-predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 1000 rounds.\n",
      "[2000]\tvalid_0's mape: 0.0155815\n",
      "[4000]\tvalid_0's mape: 0.0112608\n",
      "[6000]\tvalid_0's mape: 0.0104065\n",
      "[8000]\tvalid_0's mape: 0.0100278\n",
      "[10000]\tvalid_0's mape: 0.00980722\n",
      "[12000]\tvalid_0's mape: 0.00965274\n",
      "[14000]\tvalid_0's mape: 0.00954385\n",
      "[16000]\tvalid_0's mape: 0.00946467\n",
      "[18000]\tvalid_0's mape: 0.00940324\n",
      "[20000]\tvalid_0's mape: 0.00935387\n",
      "[22000]\tvalid_0's mape: 0.00931445\n",
      "[24000]\tvalid_0's mape: 0.00928176\n",
      "[26000]\tvalid_0's mape: 0.00925367\n",
      "[28000]\tvalid_0's mape: 0.00923029\n",
      "[30000]\tvalid_0's mape: 0.00921081\n",
      "[32000]\tvalid_0's mape: 0.00919438\n",
      "[34000]\tvalid_0's mape: 0.00918019\n",
      "[36000]\tvalid_0's mape: 0.00916695\n",
      "[38000]\tvalid_0's mape: 0.00915597\n",
      "[40000]\tvalid_0's mape: 0.00914638\n",
      "[42000]\tvalid_0's mape: 0.00913796\n",
      "[44000]\tvalid_0's mape: 0.0091303\n",
      "[46000]\tvalid_0's mape: 0.00912313\n",
      "[48000]\tvalid_0's mape: 0.00911667\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "\n",
    "predicts_val = []\n",
    "predicts_test = test[['building_id']].copy()\n",
    "best_iterations = []\n",
    "scores = []\n",
    "for i, id_fold in enumerate(id_cv):\n",
    "    #id_fold = id_fold['building_id']\n",
    "    # split train, val\n",
    "    df_train = train.loc[~train['building_id'].isin(id_fold)].copy()\n",
    "    df_val = train.loc[train['building_id'].isin(id_fold)].copy()\n",
    "    df_test = test.copy()\n",
    "    \n",
    "    # feat eng\n",
    "    feat_eng = FeatureEngineering(cols_num, cols_cat, cols_feats, col_target)\n",
    "    df_train = feat_eng.fit_transform(df_train)\n",
    "    df_val = feat_eng.transform(df_val)\n",
    "    df_test = feat_eng.transform(df_test)\n",
    "\n",
    "    # Construct lgb dataset\n",
    "    lgb_train = lgb.Dataset(df_train[cols_feats], df_train[col_target]).construct()\n",
    "    lgb_val = lgb.Dataset(df_val[cols_feats], df_val[col_target], reference=lgb_train).construct()\n",
    "        \n",
    "    # train model    \n",
    "    model = lgb.train(params, lgb_train, valid_sets=lgb_val, **lgb_other_params, keep_training_booster=True)\n",
    "    best_iterations.append(model.best_iteration)\n",
    "\n",
    "    # Predict on val set\n",
    "    y_pred = model.predict(df_val[cols_feats])\n",
    "    mse = mean_squared_error(df_val[col_target], y_pred)\n",
    "    mae = mean_absolute_error(df_val[col_target], y_pred)\n",
    "    mape = cal_mape(df_val[col_target], y_pred)\n",
    "    \n",
    "    y_pred_final = post_processing(y_pred, df_val)\n",
    "    score = cal_score(df_val['total_price'], y_pred_final)\n",
    "    scores.append(score)\n",
    "\n",
    "    print('(Fold %d) score, mse, mae, mape: %.5f, %.5f, %.5f, %.5f.' % (i, score, mse, mae, mape))\n",
    "    predicts_val.append(pd.DataFrame({'building_id': df_val['building_id'],\n",
    "                                      'total_price': df_val['total_price'],\n",
    "                                      'total_price_predict': y_pred_final}))\n",
    "\n",
    "    # Predict on test set\n",
    "    y_pred_test = model.predict(df_test[cols_feats])\n",
    "    predicts_test['target_{}'.format(i)] = y_pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Total run time:', time.time() - t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Kfold cv score (mean, stdev):', np.mean(scores), np.std(scores)) # 5868.87314"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicts_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat(predicts_val, axis=0).to_csv('output/'+model_name+'-cv.csv', index=False)\n",
    "\n",
    "predicts_test['target_mean'] = \\\n",
    "    predicts_test[['target_'+str(i) for i in range(len(id_cv))]].mean(axis=1)\n",
    "predicts_test['total_price'] = post_processing(predicts_test['target_mean'].values, df_test)\n",
    "predicts_test[['building_id', 'total_price']].to_csv('output/'+model_name+'-test-kfold.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Best iterations:', best_iterations)\n",
    "print('Mean best iterations:', int(np.mean(best_iterations)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One-fold test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lgb_other_params_one = {'num_boost_round': int(np.mean(best_iterations)*1.15),\n",
    "                        'verbose_eval': 2000,}\n",
    "lgb_other_params_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = train.copy()\n",
    "df_test = test.copy()\n",
    "\n",
    "# feat eng\n",
    "feat_eng = FeatureEngineering(cols_num, cols_cat, cols_feats, col_target)\n",
    "df_train = feat_eng.fit_transform(df_train)\n",
    "df_test = feat_eng.transform(df_test)\n",
    "\n",
    "# Construct lgb dataset\n",
    "lgb_train = lgb.Dataset(df_train[cols_feats], df_train[col_target]).construct()\n",
    "\n",
    "# train model    \n",
    "model = lgb.train(params, lgb_train, valid_sets=lgb_train, **lgb_other_params_one, keep_training_booster=True)\n",
    "\n",
    "y_pred = model.predict(df_test[cols_feats])\n",
    "y_pred_final = post_processing(y_pred, df_test)\n",
    "\n",
    "df_test_pred = pd.DataFrame({'building_id': df_test['building_id'], \n",
    "                             'total_price': y_pred_final})\n",
    "display(df_test_pred.head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_pred.to_csv('output/'+model_name+'-test-one.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb.plot_importance(model, figsize=(12, 30), height=0.5) # 050, 051, 052\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb.plot_importance(model, figsize=(12, 30), height=0.5, importance_type='gain') # 050, 051, 052\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "193.785px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}