stack_mean_11_1-11,14-17.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosave disabled\n"
     ]
    }
   ],
   "source": [
    "%autosave 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "#from sklearn.linear_model import Lasso, LassoCV, LinearRegression\n",
    "#from sklearn.model_selection import KFold\n",
    "from utilities import cal_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack_idx = '11'\n",
    "models = '1-11,14-17'\n",
    "use_test_kfold = set([2, 7, 8])\n",
    "\n",
    "#is_per_area = False\n",
    "#is_log = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read CV predictions and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_models(exp):\n",
    "    exp_split = exp.split(',')\n",
    "    idx_models = []\n",
    "    for e in exp_split:\n",
    "        if '-' in e:\n",
    "            n0, n1 = e.split('-')\n",
    "            idx_models.extend(list(range(int(n0), int(n1)+1, 1)))\n",
    "        else:\n",
    "            idx_models.append(int(e))\n",
    "    return idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n"
     ]
    }
   ],
   "source": [
    "idx_models = parse_models(models)\n",
    "print(idx_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_in_output = [f for f in os.listdir('output/') if os.path.isfile('output/'+f)]\n",
    "files_cv = {idx: [f for f in files_in_output \n",
    "                  if f.startswith('model-%02d-' % idx) and f.endswith('cv.csv')][0] \n",
    "            for idx in idx_models}\n",
    "files_test_one = {idx: [f for f in files_in_output \n",
    "                        if f.startswith('model-%02d-' % idx) and f.endswith('test-one.csv')][0]\n",
    "                  for idx in idx_models}\n",
    "files_test_kf = {idx: [f for f in files_in_output \n",
    "                       if f.startswith('model-%02d-' % idx) and f.endswith('test-kfold.csv')][0]\n",
    "                 for idx in idx_models}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1 model-01-lgb-cv.csv\n",
      " 1 model-01-lgb-test-kfold.csv\n",
      " 1 model-01-lgb-test-one.csv\n",
      " 2 model-02-keras-search-cv.csv\n",
      " 2 model-02-keras-search-test-kfold.csv\n",
      " 2 model-02-keras-search-test-one.csv\n",
      " 3 model-03-lgb-feats-selection-cv.csv\n",
      " 3 model-03-lgb-feats-selection-test-kfold.csv\n",
      " 3 model-03-lgb-feats-selection-test-one.csv\n",
      " 4 model-04-lgb-PCA-cv.csv\n",
      " 4 model-04-lgb-PCA-test-kfold.csv\n",
      " 4 model-04-lgb-PCA-test-one.csv\n",
      " 5 model-05-lgb-wo-per-area-cv.csv\n",
      " 5 model-05-lgb-wo-per-area-test-kfold.csv\n",
      " 5 model-05-lgb-wo-per-area-test-one.csv\n",
      " 6 model-06-lgb-lr0.001-cv.csv\n",
      " 6 model-06-lgb-lr0.001-test-kfold.csv\n",
      " 6 model-06-lgb-lr0.001-test-one.csv\n",
      " 7 model-07-keras-embedding-cv.csv\n",
      " 7 model-07-keras-embedding-test-kfold.csv\n",
      " 7 model-07-keras-embedding-test-one.csv\n",
      " 8 model-08-keras-search-long-cv.csv\n",
      " 8 model-08-keras-search-long-test-kfold.csv\n",
      " 8 model-08-keras-search-long-test-one.csv\n",
      " 9 model-09-lgb-feats-selection-75-cv.csv\n",
      " 9 model-09-lgb-feats-selection-75-test-kfold.csv\n",
      " 9 model-09-lgb-feats-selection-75-test-one.csv\n",
      "10 model-10-lgb-feats-selection-75-lr-0.001-cv.csv\n",
      "10 model-10-lgb-feats-selection-75-lr-0.001-test-kfold.csv\n",
      "10 model-10-lgb-feats-selection-75-lr-0.001-test-one.csv\n",
      "11 model-11-rf-cv.csv\n",
      "11 model-11-rf-test-kfold.csv\n",
      "11 model-11-rf-test-one.csv\n",
      "14 model-14-lgb-feats-selection-75-lr-0.001-rand-cv.csv\n",
      "14 model-14-lgb-feats-selection-75-lr-0.001-rand-test-kfold.csv\n",
      "14 model-14-lgb-feats-selection-75-lr-0.001-rand-test-one.csv\n",
      "15 model-15-lgb-feats-selection-75-lr-0.001-rand323-cv.csv\n",
      "15 model-15-lgb-feats-selection-75-lr-0.001-rand323-test-kfold.csv\n",
      "15 model-15-lgb-feats-selection-75-lr-0.001-rand323-test-one.csv\n",
      "16 model-16-lgb-feats-selection-68-lr-0.001-mix5-cv.csv\n",
      "16 model-16-lgb-feats-selection-68-lr-0.001-mix5-test-kfold.csv\n",
      "16 model-16-lgb-feats-selection-68-lr-0.001-mix5-test-one.csv\n",
      "17 model-17-lgb-feats-selection-70-lr-0.001-mix5-cv.csv\n",
      "17 model-17-lgb-feats-selection-70-lr-0.001-mix5-test-kfold.csv\n",
      "17 model-17-lgb-feats-selection-70-lr-0.001-mix5-test-one.csv\n"
     ]
    }
   ],
   "source": [
    "for k in files_cv: \n",
    "    print('%2d'%k, files_cv[k])\n",
    "    print('%2d'%k, files_test_kf[k])\n",
    "    print('%2d'%k, files_test_one[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('dataset/train.csv')\n",
    "df_test = pd.read_csv('dataset/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV predictions:\n",
      "No. 0 file: model-01-lgb-cv.csv\n",
      "No. 1 file: model-02-keras-search-cv.csv\n",
      "No. 2 file: model-03-lgb-feats-selection-cv.csv\n",
      "No. 3 file: model-04-lgb-PCA-cv.csv\n",
      "No. 4 file: model-05-lgb-wo-per-area-cv.csv\n",
      "No. 5 file: model-06-lgb-lr0.001-cv.csv\n",
      "No. 6 file: model-07-keras-embedding-cv.csv\n",
      "No. 7 file: model-08-keras-search-long-cv.csv\n",
      "No. 8 file: model-09-lgb-feats-selection-75-cv.csv\n",
      "No. 9 file: model-10-lgb-feats-selection-75-lr-0.001-cv.csv\n",
      "No. 10 file: model-11-rf-cv.csv\n",
      "No. 11 file: model-14-lgb-feats-selection-75-lr-0.001-rand-cv.csv\n",
      "No. 12 file: model-15-lgb-feats-selection-75-lr-0.001-rand323-cv.csv\n",
      "No. 13 file: model-16-lgb-feats-selection-68-lr-0.001-mix5-cv.csv\n",
      "No. 14 file: model-17-lgb-feats-selection-70-lr-0.001-mix5-cv.csv\n",
      "Test predictions:\n",
      "No. 0 file: model-01-lgb-test-one.csv\n",
      "No. 1 file: model-02-keras-search-test-kfold.csv\n",
      "No. 2 file: model-03-lgb-feats-selection-test-one.csv\n",
      "No. 3 file: model-04-lgb-PCA-test-one.csv\n",
      "No. 4 file: model-05-lgb-wo-per-area-test-one.csv\n",
      "No. 5 file: model-06-lgb-lr0.001-test-one.csv\n",
      "No. 6 file: model-07-keras-embedding-test-kfold.csv\n",
      "No. 7 file: model-08-keras-search-long-test-kfold.csv\n",
      "No. 8 file: model-09-lgb-feats-selection-75-test-one.csv\n",
      "No. 9 file: model-10-lgb-feats-selection-75-lr-0.001-test-one.csv\n",
      "No. 10 file: model-11-rf-test-one.csv\n",
      "No. 11 file: model-14-lgb-feats-selection-75-lr-0.001-rand-test-one.csv\n",
      "No. 12 file: model-15-lgb-feats-selection-75-lr-0.001-rand323-test-one.csv\n",
      "No. 13 file: model-16-lgb-feats-selection-68-lr-0.001-mix5-test-one.csv\n",
      "No. 14 file: model-17-lgb-feats-selection-70-lr-0.001-mix5-test-one.csv\n"
     ]
    }
   ],
   "source": [
    "cv = df_train[['building_id', 'building_area', 'total_price']]\n",
    "test = df_test[['building_id', 'building_area']]\n",
    "\n",
    "print('CV predictions:')\n",
    "for i, idx_model in enumerate(idx_models):\n",
    "    f = files_cv[idx_model]\n",
    "    print('No. {} file: {}'.format(i, f))\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    \n",
    "    cv = pd.merge(cv, df[['building_id', 'total_price_predict']], on='building_id')\n",
    "    \n",
    "    cv = cv.rename(columns = {'total_price_predict': 'pred_{}'.format(idx_model)})\n",
    "    cv[f'log_pred_{idx_model}'] = np.log1p(cv[f'pred_{idx_model}'])\n",
    "    cv[f'log_parea_pred_{idx_model}'] = np.log1p( cv[f'pred_{idx_model}'] / cv['building_area'] )\n",
    "\n",
    "cv['log_total_price'] = np.log1p(cv['total_price'])\n",
    "cv['log_parea_total_price'] = np.log1p( cv['total_price'] / cv['building_area'] )\n",
    "\n",
    "print('Test predictions:')\n",
    "for i, idx in enumerate(idx_models):\n",
    "    f = files_test_kf[idx] if idx in use_test_kfold else files_test_one[idx]\n",
    "    print('No. {} file: {}'.format(i, f))\n",
    "    df = pd.read_csv('output/'+f)\n",
    "\n",
    "    test = pd.merge(test, df[['building_id','total_price']], on='building_id')\n",
    "        \n",
    "    test = test.rename(columns = {'total_price': 'pred_{}'.format(idx)})\n",
    "    test[f'log_pred_{idx}'] = np.log1p(test[f'pred_{idx}'])\n",
    "    test[f'log_parea_pred_{idx}'] = np.log1p( test[f'pred_{idx}'] / test['building_area'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>building_area</th>\n",
       "      <th>total_price</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>log_parea_pred_1</th>\n",
       "      <th>pred_2</th>\n",
       "      <th>log_pred_2</th>\n",
       "      <th>log_parea_pred_2</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>...</th>\n",
       "      <th>log_pred_15</th>\n",
       "      <th>log_parea_pred_15</th>\n",
       "      <th>pred_16</th>\n",
       "      <th>log_pred_16</th>\n",
       "      <th>log_parea_pred_16</th>\n",
       "      <th>pred_17</th>\n",
       "      <th>log_pred_17</th>\n",
       "      <th>log_parea_pred_17</th>\n",
       "      <th>log_total_price</th>\n",
       "      <th>log_parea_total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e3mMIMR3JJqCaXz1</td>\n",
       "      <td>3.418175</td>\n",
       "      <td>6.476038e+05</td>\n",
       "      <td>6.331552e+05</td>\n",
       "      <td>13.358472</td>\n",
       "      <td>12.129369</td>\n",
       "      <td>717209.94</td>\n",
       "      <td>13.483125</td>\n",
       "      <td>12.254022</td>\n",
       "      <td>6.656891e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>13.411432</td>\n",
       "      <td>12.182328</td>\n",
       "      <td>6.542462e+05</td>\n",
       "      <td>13.391241</td>\n",
       "      <td>12.162137</td>\n",
       "      <td>6.552449e+05</td>\n",
       "      <td>13.392766</td>\n",
       "      <td>12.163663</td>\n",
       "      <td>13.381036</td>\n",
       "      <td>12.151933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LgwzgklNvy4QCtq5</td>\n",
       "      <td>4.041309</td>\n",
       "      <td>3.321452e+06</td>\n",
       "      <td>3.064324e+06</td>\n",
       "      <td>14.935338</td>\n",
       "      <td>13.538770</td>\n",
       "      <td>2899842.20</td>\n",
       "      <td>14.880167</td>\n",
       "      <td>13.483600</td>\n",
       "      <td>3.079196e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>14.963920</td>\n",
       "      <td>13.567353</td>\n",
       "      <td>3.128284e+06</td>\n",
       "      <td>14.955995</td>\n",
       "      <td>13.559428</td>\n",
       "      <td>3.130186e+06</td>\n",
       "      <td>14.956603</td>\n",
       "      <td>13.560036</td>\n",
       "      <td>15.015913</td>\n",
       "      <td>13.619345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ucIR2NLLsC3T650L</td>\n",
       "      <td>5.584279</td>\n",
       "      <td>9.570885e+06</td>\n",
       "      <td>9.827776e+06</td>\n",
       "      <td>16.100723</td>\n",
       "      <td>14.380769</td>\n",
       "      <td>9766813.00</td>\n",
       "      <td>16.094501</td>\n",
       "      <td>14.374546</td>\n",
       "      <td>9.814852e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>16.091430</td>\n",
       "      <td>14.371475</td>\n",
       "      <td>9.758797e+06</td>\n",
       "      <td>16.093680</td>\n",
       "      <td>14.373725</td>\n",
       "      <td>9.787304e+06</td>\n",
       "      <td>16.096597</td>\n",
       "      <td>14.376642</td>\n",
       "      <td>16.074236</td>\n",
       "      <td>14.354282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jre1pJhcQj91Kdky</td>\n",
       "      <td>13.563031</td>\n",
       "      <td>1.421501e+07</td>\n",
       "      <td>1.255350e+07</td>\n",
       "      <td>16.345510</td>\n",
       "      <td>13.738164</td>\n",
       "      <td>12699800.00</td>\n",
       "      <td>16.357097</td>\n",
       "      <td>13.749750</td>\n",
       "      <td>1.255981e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>16.351229</td>\n",
       "      <td>13.743883</td>\n",
       "      <td>1.266868e+07</td>\n",
       "      <td>16.354644</td>\n",
       "      <td>13.747297</td>\n",
       "      <td>1.270893e+07</td>\n",
       "      <td>16.357815</td>\n",
       "      <td>13.750469</td>\n",
       "      <td>16.469809</td>\n",
       "      <td>13.862462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rQpYpY9nRG7X5mmr</td>\n",
       "      <td>4.688108</td>\n",
       "      <td>7.627120e+05</td>\n",
       "      <td>1.215194e+06</td>\n",
       "      <td>14.010415</td>\n",
       "      <td>12.465389</td>\n",
       "      <td>2012610.50</td>\n",
       "      <td>14.514944</td>\n",
       "      <td>12.969916</td>\n",
       "      <td>1.128419e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>13.923267</td>\n",
       "      <td>12.378241</td>\n",
       "      <td>1.122232e+06</td>\n",
       "      <td>13.930831</td>\n",
       "      <td>12.385805</td>\n",
       "      <td>1.110012e+06</td>\n",
       "      <td>13.919882</td>\n",
       "      <td>12.374856</td>\n",
       "      <td>13.544637</td>\n",
       "      <td>11.999613</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id  building_area   total_price        pred_1  log_pred_1  \\\n",
       "0  e3mMIMR3JJqCaXz1       3.418175  6.476038e+05  6.331552e+05   13.358472   \n",
       "1  LgwzgklNvy4QCtq5       4.041309  3.321452e+06  3.064324e+06   14.935338   \n",
       "2  ucIR2NLLsC3T650L       5.584279  9.570885e+06  9.827776e+06   16.100723   \n",
       "3  jre1pJhcQj91Kdky      13.563031  1.421501e+07  1.255350e+07   16.345510   \n",
       "4  rQpYpY9nRG7X5mmr       4.688108  7.627120e+05  1.215194e+06   14.010415   \n",
       "\n",
       "   log_parea_pred_1       pred_2  log_pred_2  log_parea_pred_2        pred_3  \\\n",
       "0         12.129369    717209.94   13.483125         12.254022  6.656891e+05   \n",
       "1         13.538770   2899842.20   14.880167         13.483600  3.079196e+06   \n",
       "2         14.380769   9766813.00   16.094501         14.374546  9.814852e+06   \n",
       "3         13.738164  12699800.00   16.357097         13.749750  1.255981e+07   \n",
       "4         12.465389   2012610.50   14.514944         12.969916  1.128419e+06   \n",
       "\n",
       "           ...            log_pred_15  log_parea_pred_15       pred_16  \\\n",
       "0          ...              13.411432          12.182328  6.542462e+05   \n",
       "1          ...              14.963920          13.567353  3.128284e+06   \n",
       "2          ...              16.091430          14.371475  9.758797e+06   \n",
       "3          ...              16.351229          13.743883  1.266868e+07   \n",
       "4          ...              13.923267          12.378241  1.122232e+06   \n",
       "\n",
       "   log_pred_16  log_parea_pred_16       pred_17  log_pred_17  \\\n",
       "0    13.391241          12.162137  6.552449e+05    13.392766   \n",
       "1    14.955995          13.559428  3.130186e+06    14.956603   \n",
       "2    16.093680          14.373725  9.787304e+06    16.096597   \n",
       "3    16.354644          13.747297  1.270893e+07    16.357815   \n",
       "4    13.930831          12.385805  1.110012e+06    13.919882   \n",
       "\n",
       "   log_parea_pred_17  log_total_price  log_parea_total_price  \n",
       "0          12.163663        13.381036              12.151933  \n",
       "1          13.560036        15.015913              13.619345  \n",
       "2          14.376642        16.074236              14.354282  \n",
       "3          13.750469        16.469809              13.862462  \n",
       "4          12.374856        13.544637              11.999613  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>building_area</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>log_parea_pred_1</th>\n",
       "      <th>pred_2</th>\n",
       "      <th>log_pred_2</th>\n",
       "      <th>log_parea_pred_2</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>log_pred_3</th>\n",
       "      <th>...</th>\n",
       "      <th>log_parea_pred_14</th>\n",
       "      <th>pred_15</th>\n",
       "      <th>log_pred_15</th>\n",
       "      <th>log_parea_pred_15</th>\n",
       "      <th>pred_16</th>\n",
       "      <th>log_pred_16</th>\n",
       "      <th>log_parea_pred_16</th>\n",
       "      <th>pred_17</th>\n",
       "      <th>log_pred_17</th>\n",
       "      <th>log_parea_pred_17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>3.418175</td>\n",
       "      <td>1.526912e+07</td>\n",
       "      <td>16.541343</td>\n",
       "      <td>15.312236</td>\n",
       "      <td>12470072.0</td>\n",
       "      <td>16.338842</td>\n",
       "      <td>15.109735</td>\n",
       "      <td>1.531685e+07</td>\n",
       "      <td>16.544464</td>\n",
       "      <td>...</td>\n",
       "      <td>15.270289</td>\n",
       "      <td>1.476388e+07</td>\n",
       "      <td>16.507694</td>\n",
       "      <td>15.278587</td>\n",
       "      <td>1.388634e+07</td>\n",
       "      <td>16.446416</td>\n",
       "      <td>15.217310</td>\n",
       "      <td>1.449905e+07</td>\n",
       "      <td>16.489594</td>\n",
       "      <td>15.260487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>7.726227</td>\n",
       "      <td>3.924241e+06</td>\n",
       "      <td>15.182684</td>\n",
       "      <td>13.138065</td>\n",
       "      <td>3916552.2</td>\n",
       "      <td>15.180723</td>\n",
       "      <td>13.136104</td>\n",
       "      <td>3.977095e+06</td>\n",
       "      <td>15.196062</td>\n",
       "      <td>...</td>\n",
       "      <td>13.144670</td>\n",
       "      <td>3.928353e+06</td>\n",
       "      <td>15.183731</td>\n",
       "      <td>13.139112</td>\n",
       "      <td>3.918650e+06</td>\n",
       "      <td>15.181258</td>\n",
       "      <td>13.136639</td>\n",
       "      <td>3.917960e+06</td>\n",
       "      <td>15.181082</td>\n",
       "      <td>13.136463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>12.170581</td>\n",
       "      <td>1.096127e+07</td>\n",
       "      <td>16.209879</td>\n",
       "      <td>13.710858</td>\n",
       "      <td>11912735.0</td>\n",
       "      <td>16.293119</td>\n",
       "      <td>13.794098</td>\n",
       "      <td>1.084967e+07</td>\n",
       "      <td>16.199646</td>\n",
       "      <td>...</td>\n",
       "      <td>13.677153</td>\n",
       "      <td>1.062900e+07</td>\n",
       "      <td>16.179096</td>\n",
       "      <td>13.680076</td>\n",
       "      <td>1.073546e+07</td>\n",
       "      <td>16.189063</td>\n",
       "      <td>13.690042</td>\n",
       "      <td>1.063343e+07</td>\n",
       "      <td>16.179513</td>\n",
       "      <td>13.680493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>2.252256</td>\n",
       "      <td>6.155550e+06</td>\n",
       "      <td>15.632865</td>\n",
       "      <td>14.820933</td>\n",
       "      <td>5940670.0</td>\n",
       "      <td>15.597333</td>\n",
       "      <td>14.785400</td>\n",
       "      <td>6.015238e+06</td>\n",
       "      <td>15.609807</td>\n",
       "      <td>...</td>\n",
       "      <td>14.815150</td>\n",
       "      <td>6.085773e+06</td>\n",
       "      <td>15.621464</td>\n",
       "      <td>14.809532</td>\n",
       "      <td>6.080935e+06</td>\n",
       "      <td>15.620669</td>\n",
       "      <td>14.808737</td>\n",
       "      <td>6.119193e+06</td>\n",
       "      <td>15.626941</td>\n",
       "      <td>14.815009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>5.813985</td>\n",
       "      <td>1.062995e+06</td>\n",
       "      <td>13.876602</td>\n",
       "      <td>12.116340</td>\n",
       "      <td>1088488.1</td>\n",
       "      <td>13.900301</td>\n",
       "      <td>12.140039</td>\n",
       "      <td>1.027248e+06</td>\n",
       "      <td>13.842395</td>\n",
       "      <td>...</td>\n",
       "      <td>12.116189</td>\n",
       "      <td>1.061918e+06</td>\n",
       "      <td>13.875588</td>\n",
       "      <td>12.115326</td>\n",
       "      <td>1.072461e+06</td>\n",
       "      <td>13.885468</td>\n",
       "      <td>12.125206</td>\n",
       "      <td>1.084776e+06</td>\n",
       "      <td>13.896885</td>\n",
       "      <td>12.136623</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id  building_area        pred_1  log_pred_1  \\\n",
       "0  X5gsdTWGS3W7JJQB       3.418175  1.526912e+07   16.541343   \n",
       "1  BTshNOJyKHnT2YIT       7.726227  3.924241e+06   15.182684   \n",
       "2  dhdymr0lV8N5kZOT      12.170581  1.096127e+07   16.209879   \n",
       "3  VEwyGGMcD56w5BOc       2.252256  6.155550e+06   15.632865   \n",
       "4  wmUeMoJZfsqaSX9b       5.813985  1.062995e+06   13.876602   \n",
       "\n",
       "   log_parea_pred_1      pred_2  log_pred_2  log_parea_pred_2        pred_3  \\\n",
       "0         15.312236  12470072.0   16.338842         15.109735  1.531685e+07   \n",
       "1         13.138065   3916552.2   15.180723         13.136104  3.977095e+06   \n",
       "2         13.710858  11912735.0   16.293119         13.794098  1.084967e+07   \n",
       "3         14.820933   5940670.0   15.597333         14.785400  6.015238e+06   \n",
       "4         12.116340   1088488.1   13.900301         12.140039  1.027248e+06   \n",
       "\n",
       "   log_pred_3        ...          log_parea_pred_14       pred_15  \\\n",
       "0   16.544464        ...                  15.270289  1.476388e+07   \n",
       "1   15.196062        ...                  13.144670  3.928353e+06   \n",
       "2   16.199646        ...                  13.677153  1.062900e+07   \n",
       "3   15.609807        ...                  14.815150  6.085773e+06   \n",
       "4   13.842395        ...                  12.116189  1.061918e+06   \n",
       "\n",
       "   log_pred_15  log_parea_pred_15       pred_16  log_pred_16  \\\n",
       "0    16.507694          15.278587  1.388634e+07    16.446416   \n",
       "1    15.183731          13.139112  3.918650e+06    15.181258   \n",
       "2    16.179096          13.680076  1.073546e+07    16.189063   \n",
       "3    15.621464          14.809532  6.080935e+06    15.620669   \n",
       "4    13.875588          12.115326  1.072461e+06    13.885468   \n",
       "\n",
       "   log_parea_pred_16       pred_17  log_pred_17  log_parea_pred_17  \n",
       "0          15.217310  1.449905e+07    16.489594          15.260487  \n",
       "1          13.136639  3.917960e+06    15.181082          13.136463  \n",
       "2          13.690042  1.063343e+07    16.179513          13.680493  \n",
       "3          14.808737  6.119193e+06    15.626941          14.815009  \n",
       "4          12.125206  1.084776e+06    13.896885          12.136623  \n",
       "\n",
       "[5 rows x 47 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check models scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_models = {idx_model: cal_score(cv['total_price'], cv[f'pred_{idx_model}'])\n",
    "                 for idx_model in idx_models}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0 model-01 5870.873059\n",
      " 1 model-02 5400.852164\n",
      " 2 model-03 5877.873452\n",
      " 3 model-04 5713.867808\n",
      " 4 model-05 5724.869598\n",
      " 5 model-06 5886.873769\n",
      " 6 model-07 5171.836449\n",
      " 7 model-08 5514.858826\n",
      " 8 model-09 5872.873118\n",
      " 9 model-10 5897.873845\n",
      "10 model-11 5075.838018\n",
      "11 model-14 5908.873901\n",
      "12 model-15 5900.873836\n",
      "13 model-16 5907.874126\n",
      "14 model-17 5905.874165\n"
     ]
    }
   ],
   "source": [
    "for i, (idx_model, s) in enumerate(scores_models.items()):\n",
    "    print('%2d'%i, 'model-%02d'%idx_model, '%.6f'%s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search subset model mean "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prefix(is_per_area, is_log):\n",
    "    if is_log:\n",
    "        if is_per_area:\n",
    "            col_prefix = 'log_parea_pred'\n",
    "        else:\n",
    "            col_prefix = 'log_pred'\n",
    "    else:\n",
    "        if is_per_area:\n",
    "            raise 'Not support'\n",
    "        else:\n",
    "            col_prefix = 'pred'\n",
    "    return col_prefix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_subset(df, idx_models, is_per_area, is_log):\n",
    "    col_prefix = get_prefix(is_per_area, is_log)\n",
    "    cols_models = [col_prefix+'_'+str(i) for i in idx_models]\n",
    "    return df[cols_models].mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_mean_subset(df, idx_models, is_per_area, is_log):\n",
    "    y_pred = mean_subset(df, idx_models, is_per_area, is_log)\n",
    "    y_pred_final = y_pred\n",
    "    if is_log:\n",
    "        y_pred_final = np.expm1(y_pred_final)\n",
    "    if is_per_area:\n",
    "        y_pred_final = y_pred_final * df['building_area']\n",
    "    return y_pred_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_mean_subset(cv, idx_models, is_per_area, is_log):\n",
    "    idx_models_subset = idx_models.copy()\n",
    "    result_list = []\n",
    "    y_pred_cv = predict_mean_subset(cv, idx_models_subset, is_per_area=is_per_area, is_log=is_log)\n",
    "    score = cal_score(cv['total_price'], y_pred_cv)\n",
    "    print('%.6f'%score, idx_models_subset)\n",
    "    result_list.append([score, idx_models_subset])\n",
    "    while len(idx_models_subset) > 1:\n",
    "        print('N models remained:', len(idx_models_subset))\n",
    "        result_list_subset = []\n",
    "        for i_rm in range(len(idx_models_subset)):\n",
    "            idx_models_subset_next = idx_models_subset[:i_rm]+idx_models_subset[i_rm+1:]\n",
    "            y_pred_cv = predict_mean_subset(cv, idx_models_subset_next, is_per_area=is_per_area, is_log=is_log)\n",
    "            score = cal_score(cv['total_price'], y_pred_cv)\n",
    "            print('%.6f'%score, idx_models_subset_next)\n",
    "            result_list_subset.append([score, idx_models_subset_next])\n",
    "        result_list.extend(result_list_subset)\n",
    "        result_list_subset.sort(key=lambda x: x[0], reverse=True)\n",
    "        print('Best of the round:', result_list_subset[0][0], result_list_subset[0][1])\n",
    "        idx_models_subset = result_list_subset[0][1]\n",
    "        print()\n",
    "    result_list.sort(key=lambda x: x[0], reverse=True)\n",
    "    return result_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Log mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5919.874467 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "N models remained: 15\n",
      "5921.874399 [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5924.874373 [1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5914.874372 [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5916.874420 [1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5917.874232 [1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5921.874386 [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5913.874387 [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5911.874336 [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 14, 15, 16, 17]\n",
      "5923.874398 [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 14, 15, 16, 17]\n",
      "5916.874371 [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 14, 15, 16, 17]\n",
      "5936.875176 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5919.874366 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15, 16, 17]\n",
      "5918.874371 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 16, 17]\n",
      "5917.874356 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 17]\n",
      "5918.874352 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16]\n",
      "Best of the round: 5936.875176322402 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "\n",
      "N models remained: 14\n",
      "5937.875168 [2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5936.875144 [1, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5937.875136 [1, 2, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5933.875153 [1, 2, 3, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5931.874989 [1, 2, 3, 4, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5937.875153 [1, 2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5928.875182 [1, 2, 3, 4, 5, 6, 8, 9, 10, 14, 15, 16, 17]\n",
      "5928.875097 [1, 2, 3, 4, 5, 6, 7, 9, 10, 14, 15, 16, 17]\n",
      "5937.875164 [1, 2, 3, 4, 5, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "5933.875135 [1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 15, 16, 17]\n",
      "5934.875130 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17]\n",
      "5935.875136 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 16, 17]\n",
      "5934.875119 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 17]\n",
      "5933.875116 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16]\n",
      "Best of the round: 5937.875168450763 [2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "\n",
      "N models remained: 13\n",
      "5936.875180 [3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5938.875102 [2, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5932.875134 [2, 3, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5936.874956 [2, 3, 4, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5938.875120 [2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5925.875221 [2, 3, 4, 5, 6, 8, 9, 10, 14, 15, 16, 17]\n",
      "5928.875120 [2, 3, 4, 5, 6, 7, 9, 10, 14, 15, 16, 17]\n",
      "5938.875134 [2, 3, 4, 5, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "5938.875106 [2, 3, 4, 5, 6, 7, 8, 9, 14, 15, 16, 17]\n",
      "5938.875100 [2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17]\n",
      "5938.875107 [2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 16, 17]\n",
      "5938.875088 [2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 17]\n",
      "5938.875085 [2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16]\n",
      "Best of the round: 5938.8751336909745 [2, 3, 4, 5, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "\n",
      "N models remained: 12\n",
      "5940.875197 [3, 4, 5, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "5940.875030 [2, 4, 5, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "5936.875081 [2, 3, 5, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "5936.874889 [2, 3, 4, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "5940.875049 [2, 3, 4, 5, 7, 8, 10, 14, 15, 16, 17]\n",
      "5928.875246 [2, 3, 4, 5, 6, 8, 10, 14, 15, 16, 17]\n",
      "5927.875125 [2, 3, 4, 5, 6, 7, 10, 14, 15, 16, 17]\n",
      "5940.875038 [2, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17]\n",
      "5939.875032 [2, 3, 4, 5, 6, 7, 8, 10, 15, 16, 17]\n",
      "5941.875039 [2, 3, 4, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "5939.875020 [2, 3, 4, 5, 6, 7, 8, 10, 14, 15, 17]\n",
      "5938.875017 [2, 3, 4, 5, 6, 7, 8, 10, 14, 15, 16]\n",
      "Best of the round: 5941.875038789146 [2, 3, 4, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "\n",
      "N models remained: 11\n",
      "5940.875173 [3, 4, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "5938.874882 [2, 4, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "5936.874956 [2, 3, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "5933.874743 [2, 3, 4, 6, 7, 8, 10, 14, 16, 17]\n",
      "5938.874902 [2, 3, 4, 5, 7, 8, 10, 14, 16, 17]\n",
      "5929.875227 [2, 3, 4, 5, 6, 8, 10, 14, 16, 17]\n",
      "5928.875082 [2, 3, 4, 5, 6, 7, 10, 14, 16, 17]\n",
      "5934.874880 [2, 3, 4, 5, 6, 7, 8, 14, 16, 17]\n",
      "5935.874872 [2, 3, 4, 5, 6, 7, 8, 10, 16, 17]\n",
      "5935.874862 [2, 3, 4, 5, 6, 7, 8, 10, 14, 17]\n",
      "5934.874859 [2, 3, 4, 5, 6, 7, 8, 10, 14, 16]\n",
      "Best of the round: 5940.875172702243 [3, 4, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "\n",
      "N models remained: 10\n",
      "5940.875098 [4, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "5937.875154 [3, 5, 6, 7, 8, 10, 14, 16, 17]\n",
      "5936.874936 [3, 4, 6, 7, 8, 10, 14, 16, 17]\n",
      "5942.875122 [3, 4, 5, 7, 8, 10, 14, 16, 17]\n",
      "5930.875173 [3, 4, 5, 6, 8, 10, 14, 16, 17]\n",
      "5927.874961 [3, 4, 5, 6, 7, 10, 14, 16, 17]\n",
      "5940.875106 [3, 4, 5, 6, 7, 8, 14, 16, 17]\n",
      "5939.875098 [3, 4, 5, 6, 7, 8, 10, 16, 17]\n",
      "5939.875087 [3, 4, 5, 6, 7, 8, 10, 14, 17]\n",
      "5940.875083 [3, 4, 5, 6, 7, 8, 10, 14, 16]\n",
      "Best of the round: 5942.875121834004 [3, 4, 5, 7, 8, 10, 14, 16, 17]\n",
      "\n",
      "N models remained: 9\n",
      "5942.874980 [4, 5, 7, 8, 10, 14, 16, 17]\n",
      "5939.875078 [3, 5, 7, 8, 10, 14, 16, 17]\n",
      "5936.874831 [3, 4, 7, 8, 10, 14, 16, 17]\n",
      "5939.875212 [3, 4, 5, 8, 10, 14, 16, 17]\n",
      "5924.874941 [3, 4, 5, 7, 10, 14, 16, 17]\n",
      "5935.874998 [3, 4, 5, 7, 8, 14, 16, 17]\n",
      "5938.874989 [3, 4, 5, 7, 8, 10, 16, 17]\n",
      "5938.874977 [3, 4, 5, 7, 8, 10, 14, 17]\n",
      "5937.874973 [3, 4, 5, 7, 8, 10, 14, 16]\n",
      "Best of the round: 5942.8749798044755 [4, 5, 7, 8, 10, 14, 16, 17]\n",
      "\n",
      "N models remained: 8\n",
      "5937.874875 [5, 7, 8, 10, 14, 16, 17]\n",
      "5927.874593 [4, 7, 8, 10, 14, 16, 17]\n",
      "5934.875201 [4, 5, 8, 10, 14, 16, 17]\n",
      "5929.874837 [4, 5, 7, 10, 14, 16, 17]\n",
      "5936.874750 [4, 5, 7, 8, 14, 16, 17]\n",
      "5934.874740 [4, 5, 7, 8, 10, 16, 17]\n",
      "5936.874728 [4, 5, 7, 8, 10, 14, 17]\n",
      "5933.874725 [4, 5, 7, 8, 10, 14, 16]\n",
      "Best of the round: 5937.874875382356 [5, 7, 8, 10, 14, 16, 17]\n",
      "\n",
      "N models remained: 7\n",
      "5924.874347 [7, 8, 10, 14, 16, 17]\n",
      "5930.875266 [5, 8, 10, 14, 16, 17]\n",
      "5925.874736 [5, 7, 10, 14, 16, 17]\n",
      "5927.874521 [5, 7, 8, 14, 16, 17]\n",
      "5926.874510 [5, 7, 8, 10, 16, 17]\n",
      "5924.874497 [5, 7, 8, 10, 14, 17]\n",
      "5926.874493 [5, 7, 8, 10, 14, 16]\n",
      "Best of the round: 5930.875265810579 [5, 8, 10, 14, 16, 17]\n",
      "\n",
      "N models remained: 6\n",
      "5936.874970 [8, 10, 14, 16, 17]\n",
      "5913.874535 [5, 10, 14, 16, 17]\n",
      "5932.875261 [5, 8, 14, 16, 17]\n",
      "5933.875248 [5, 8, 10, 16, 17]\n",
      "5935.875223 [5, 8, 10, 14, 17]\n",
      "5932.875216 [5, 8, 10, 14, 16]\n",
      "Best of the round: 5936.874970078768 [8, 10, 14, 16, 17]\n",
      "\n",
      "N models remained: 5\n",
      "5910.874090 [10, 14, 16, 17]\n",
      "5932.874903 [8, 14, 16, 17]\n",
      "5934.874891 [8, 10, 16, 17]\n",
      "5933.874846 [8, 10, 14, 17]\n",
      "5933.874836 [8, 10, 14, 16]\n",
      "Best of the round: 5934.874891281447 [8, 10, 16, 17]\n",
      "\n",
      "N models remained: 4\n",
      "5913.874117 [10, 16, 17]\n",
      "5920.874487 [8, 16, 17]\n",
      "5916.874433 [8, 10, 17]\n",
      "5922.874418 [8, 10, 16]\n",
      "Best of the round: 5922.874418381022 [8, 10, 16]\n",
      "\n",
      "N models remained: 3\n",
      "5909.874045 [10, 16]\n",
      "5885.872540 [8, 16]\n",
      "5880.872459 [8, 10]\n",
      "Best of the round: 5909.874044790506 [10, 16]\n",
      "\n",
      "N models remained: 2\n",
      "5907.874126 [16]\n",
      "5897.873845 [10]\n",
      "Best of the round: 5907.874126200278 [16]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result_list_log = search_mean_subset(cv, idx_models, is_per_area=False, is_log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[5942.875121834004, [3, 4, 5, 7, 8, 10, 14, 16, 17]],\n",
       " [5942.8749798044755, [4, 5, 7, 8, 10, 14, 16, 17]],\n",
       " [5941.875038789146, [2, 3, 4, 5, 6, 7, 8, 10, 14, 16, 17]],\n",
       " [5940.87519730079, [3, 4, 5, 6, 7, 8, 10, 14, 15, 16, 17]],\n",
       " [5940.875172702243, [3, 4, 5, 6, 7, 8, 10, 14, 16, 17]],\n",
       " [5940.875106267237, [3, 4, 5, 6, 7, 8, 14, 16, 17]],\n",
       " [5940.875098400611, [4, 5, 6, 7, 8, 10, 14, 16, 17]],\n",
       " [5940.875083220059, [3, 4, 5, 6, 7, 8, 10, 14, 16]],\n",
       " [5940.875048671748, [2, 3, 4, 5, 7, 8, 10, 14, 15, 16, 17]],\n",
       " [5940.875038001536, [2, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17]]]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(result_list_log[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score and subset: 5942.875122 [3, 4, 5, 7, 8, 10, 14, 16, 17]\n",
      "model-03 5877.873452\n",
      "model-04 5713.867808\n",
      "model-05 5724.869598\n",
      "model-07 5171.836449\n",
      "model-08 5514.858826\n",
      "model-10 5897.873845\n",
      "model-14 5908.873901\n",
      "model-16 5907.874126\n",
      "model-17 5905.874165\n"
     ]
    }
   ],
   "source": [
    "best_models_subset_log = result_list_log[0][1]\n",
    "print('Best score and subset:', '%.6f'%result_list_log[0][0], best_models_subset_log)\n",
    "for idx_model in best_models_subset_log:\n",
    "    print('model-%02d'%idx_model, '%.6f'%scores_models[idx_model])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# per area or not for the mean ensemble model are the same\n",
    "#result_list_per_area_log = search_mean_subset(cv, idx_models, is_per_area=True, is_log=True)\n",
    "#result_list_per_area_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Linear mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5916.873085 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "N models remained: 15\n",
      "5916.872927 [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5923.873096 [1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5915.872900 [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5912.872976 [1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5915.872784 [1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5919.872912 [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5913.873733 [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 14, 15, 16, 17]\n",
      "5910.872969 [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 14, 15, 16, 17]\n",
      "5920.872926 [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 14, 15, 16, 17]\n",
      "5916.872895 [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 14, 15, 16, 17]\n",
      "5931.873926 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5915.872891 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15, 16, 17]\n",
      "5917.872897 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 16, 17]\n",
      "5918.872881 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 17]\n",
      "5916.872877 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 16]\n",
      "Best of the round: 5931.873926118575 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "\n",
      "N models remained: 14\n",
      "5934.873838 [2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5930.874015 [1, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5936.873805 [1, 2, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5926.873856 [1, 2, 3, 5, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5925.873679 [1, 2, 3, 4, 6, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5937.873818 [1, 2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5926.874709 [1, 2, 3, 4, 5, 6, 8, 9, 10, 14, 15, 16, 17]\n",
      "5920.873872 [1, 2, 3, 4, 5, 6, 7, 9, 10, 14, 15, 16, 17]\n",
      "5935.873831 [1, 2, 3, 4, 5, 6, 7, 8, 10, 14, 15, 16, 17]\n",
      "5932.873801 [1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 15, 16, 17]\n",
      "5932.873797 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17]\n",
      "5933.873803 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 16, 17]\n",
      "5934.873785 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 17]\n",
      "5934.873781 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14, 15, 16]\n",
      "Best of the round: 5937.873818359761 [1, 2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "\n",
      "N models remained: 13\n",
      "5938.873694 [2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5931.873942 [1, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5936.873660 [1, 2, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5928.873727 [1, 2, 3, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5925.873533 [1, 2, 3, 4, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5932.874693 [1, 2, 3, 4, 5, 8, 9, 10, 14, 15, 16, 17]\n",
      "5924.873781 [1, 2, 3, 4, 5, 7, 9, 10, 14, 15, 16, 17]\n",
      "5935.873687 [1, 2, 3, 4, 5, 7, 8, 10, 14, 15, 16, 17]\n",
      "5934.873659 [1, 2, 3, 4, 5, 7, 8, 9, 14, 15, 16, 17]\n",
      "5937.873654 [1, 2, 3, 4, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "5936.873660 [1, 2, 3, 4, 5, 7, 8, 9, 10, 14, 16, 17]\n",
      "5935.873642 [1, 2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 17]\n",
      "5933.873638 [1, 2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16]\n",
      "Best of the round: 5938.873694147015 [2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "\n",
      "N models remained: 12\n",
      "5937.873865 [3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5936.873483 [2, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5928.873571 [2, 3, 5, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5932.873359 [2, 3, 4, 7, 8, 9, 10, 14, 15, 16, 17]\n",
      "5930.874683 [2, 3, 4, 5, 8, 9, 10, 14, 15, 16, 17]\n",
      "5926.873682 [2, 3, 4, 5, 7, 9, 10, 14, 15, 16, 17]\n",
      "5935.873514 [2, 3, 4, 5, 7, 8, 10, 14, 15, 16, 17]\n",
      "5936.873487 [2, 3, 4, 5, 7, 8, 9, 14, 15, 16, 17]\n",
      "5939.873482 [2, 3, 4, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "5938.873489 [2, 3, 4, 5, 7, 8, 9, 10, 14, 16, 17]\n",
      "5937.873468 [2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 17]\n",
      "5937.873465 [2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16]\n",
      "Best of the round: 5939.873481553874 [2, 3, 4, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "\n",
      "N models remained: 11\n",
      "5940.873714 [3, 4, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "5934.873195 [2, 4, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "5938.873314 [2, 3, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "5930.873078 [2, 3, 4, 7, 8, 9, 10, 15, 16, 17]\n",
      "5932.874614 [2, 3, 4, 5, 8, 9, 10, 15, 16, 17]\n",
      "5928.873506 [2, 3, 4, 5, 7, 9, 10, 15, 16, 17]\n",
      "5937.873230 [2, 3, 4, 5, 7, 8, 10, 15, 16, 17]\n",
      "5933.873192 [2, 3, 4, 5, 7, 8, 9, 15, 16, 17]\n",
      "5934.873193 [2, 3, 4, 5, 7, 8, 9, 10, 16, 17]\n",
      "5934.873173 [2, 3, 4, 5, 7, 8, 9, 10, 15, 17]\n",
      "5932.873169 [2, 3, 4, 5, 7, 8, 9, 10, 15, 16]\n",
      "Best of the round: 5940.873714031534 [3, 4, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "\n",
      "N models remained: 10\n",
      "5940.873495 [4, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "5935.873596 [3, 5, 7, 8, 9, 10, 15, 16, 17]\n",
      "5930.873358 [3, 4, 7, 8, 9, 10, 15, 16, 17]\n",
      "5934.874821 [3, 4, 5, 8, 9, 10, 15, 16, 17]\n",
      "5926.873509 [3, 4, 5, 7, 9, 10, 15, 16, 17]\n",
      "5943.873534 [3, 4, 5, 7, 8, 10, 15, 16, 17]\n",
      "5940.873503 [3, 4, 5, 7, 8, 9, 15, 16, 17]\n",
      "5939.873504 [3, 4, 5, 7, 8, 9, 10, 16, 17]\n",
      "5937.873482 [3, 4, 5, 7, 8, 9, 10, 15, 17]\n",
      "5937.873478 [3, 4, 5, 7, 8, 9, 10, 15, 16]\n",
      "Best of the round: 5943.873533816494 [3, 4, 5, 7, 8, 10, 15, 16, 17]\n",
      "\n",
      "N models remained: 9\n",
      "5940.873220 [4, 5, 7, 8, 10, 15, 16, 17]\n",
      "5936.873368 [3, 5, 7, 8, 10, 15, 16, 17]\n",
      "5932.873094 [3, 4, 7, 8, 10, 15, 16, 17]\n",
      "5936.874849 [3, 4, 5, 8, 10, 15, 16, 17]\n",
      "5926.873341 [3, 4, 5, 7, 10, 15, 16, 17]\n",
      "5935.873233 [3, 4, 5, 7, 8, 15, 16, 17]\n",
      "5936.873235 [3, 4, 5, 7, 8, 10, 16, 17]\n",
      "5938.873211 [3, 4, 5, 7, 8, 10, 15, 17]\n",
      "5937.873208 [3, 4, 5, 7, 8, 10, 15, 16]\n",
      "Best of the round: 5940.8732204836515 [4, 5, 7, 8, 10, 15, 16, 17]\n",
      "\n",
      "N models remained: 8\n",
      "5931.872967 [5, 7, 8, 10, 15, 16, 17]\n",
      "5928.872641 [4, 7, 8, 10, 15, 16, 17]\n",
      "5937.874802 [4, 5, 8, 10, 15, 16, 17]\n",
      "5931.873029 [4, 5, 7, 10, 15, 16, 17]\n",
      "5938.872772 [4, 5, 7, 8, 15, 16, 17]\n",
      "5938.872774 [4, 5, 7, 8, 10, 16, 17]\n",
      "5933.872750 [4, 5, 7, 8, 10, 15, 17]\n",
      "5934.872747 [4, 5, 7, 8, 10, 15, 16]\n",
      "Best of the round: 5938.872774309543 [4, 5, 7, 8, 10, 16, 17]\n",
      "\n",
      "N models remained: 7\n",
      "5928.872357 [5, 7, 8, 10, 16, 17]\n",
      "5921.871980 [4, 7, 8, 10, 16, 17]\n",
      "5935.874727 [4, 5, 8, 10, 16, 17]\n",
      "5928.872591 [4, 5, 7, 10, 16, 17]\n",
      "5921.872041 [4, 5, 7, 8, 16, 17]\n",
      "5919.872023 [4, 5, 7, 8, 10, 17]\n",
      "5921.872020 [4, 5, 7, 8, 10, 16]\n",
      "Best of the round: 5935.8747267182725 [4, 5, 8, 10, 16, 17]\n",
      "\n",
      "N models remained: 6\n",
      "5932.874817 [5, 8, 10, 16, 17]\n",
      "5931.874391 [4, 8, 10, 16, 17]\n",
      "5913.874405 [4, 5, 10, 16, 17]\n",
      "5932.874504 [4, 5, 8, 16, 17]\n",
      "5930.874479 [4, 5, 8, 10, 17]\n",
      "5931.874472 [4, 5, 8, 10, 16]\n",
      "Best of the round: 5932.874817027132 [5, 8, 10, 16, 17]\n",
      "\n",
      "N models remained: 5\n",
      "5934.874450 [8, 10, 16, 17]\n",
      "5907.874465 [5, 10, 16, 17]\n",
      "5932.874560 [5, 8, 16, 17]\n",
      "5934.874529 [5, 8, 10, 17]\n",
      "5932.874519 [5, 8, 10, 16]\n",
      "Best of the round: 5934.874528638981 [5, 8, 10, 17]\n",
      "\n",
      "N models remained: 4\n",
      "5918.873897 [8, 10, 17]\n",
      "5901.874328 [5, 10, 17]\n",
      "5910.873697 [5, 8, 17]\n",
      "5908.873651 [5, 8, 10]\n",
      "Best of the round: 5918.873897313872 [8, 10, 17]\n",
      "\n",
      "N models remained: 3\n",
      "5906.874070 [10, 17]\n",
      "5882.871928 [8, 17]\n",
      "5879.871833 [8, 10]\n",
      "Best of the round: 5906.874069510736 [10, 17]\n",
      "\n",
      "N models remained: 2\n",
      "5905.874165 [17]\n",
      "5897.873845 [10]\n",
      "Best of the round: 5905.874164864382 [17]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result_list_none = search_mean_subset(cv, idx_models, is_per_area=False, is_log=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[5943.873533816494, [3, 4, 5, 7, 8, 10, 15, 16, 17]],\n",
       " [5940.873714031534, [3, 4, 5, 7, 8, 9, 10, 15, 16, 17]],\n",
       " [5940.873502974041, [3, 4, 5, 7, 8, 9, 15, 16, 17]],\n",
       " [5940.87349497661, [4, 5, 7, 8, 9, 10, 15, 16, 17]],\n",
       " [5940.8732204836515, [4, 5, 7, 8, 10, 15, 16, 17]],\n",
       " [5939.873504113526, [3, 4, 5, 7, 8, 9, 10, 16, 17]],\n",
       " [5939.873481553874, [2, 3, 4, 5, 7, 8, 9, 10, 15, 16, 17]],\n",
       " [5938.873694147015, [2, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17]],\n",
       " [5938.873488950203, [2, 3, 4, 5, 7, 8, 9, 10, 14, 16, 17]],\n",
       " [5938.873313590779, [2, 3, 5, 7, 8, 9, 10, 15, 16, 17]]]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(result_list_none[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score and subset: 5943.873534 [3, 4, 5, 7, 8, 10, 15, 16, 17]\n",
      "model-03 5877.873452\n",
      "model-04 5713.867808\n",
      "model-05 5724.869598\n",
      "model-07 5171.836449\n",
      "model-08 5514.858826\n",
      "model-10 5897.873845\n",
      "model-15 5900.873836\n",
      "model-16 5907.874126\n",
      "model-17 5905.874165\n"
     ]
    }
   ],
   "source": [
    "best_models_subset_none = result_list_none[0][1]\n",
    "print('Best score and subset:', '%.6f'%result_list_none[0][0], best_models_subset_none)\n",
    "for idx_model in best_models_subset_none:\n",
    "    print('model-%02d'%idx_model, '%.6f'%scores_models[idx_model])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission for mean log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict for mean log with [3, 4, 5, 7, 8, 10, 14, 16, 17]\n"
     ]
    }
   ],
   "source": [
    "print('Predict for mean log with', best_models_subset_log)\n",
    "test['total_price'] = predict_mean_subset(test, best_models_subset_log, is_per_area=False, is_log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6571: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHXJJREFUeJzt3Xt4VfWd7/H3FwwGhKJATat4JpRqT/CGEvBuQ1UutQ/YKS0MPVbaIrVVj04dWnpanWrtc6g5dYTWSzkpx3FsRIvVw/PAGRFrqjP1BhQvGEfUZmpEo2IFoqAEvuePtfZyJyRk72Svvfbl83qePNnr/v2Fzf7u3++31u9n7o6IiAjAgKQDEBGRwqGkICIiESUFERGJKCmIiEhESUFERCJKCiIiElFSEBGRiJKCiIhElBRERCRyUNIBZGvUqFFeXV2d9XHvvfcehxxySO4DKnAqd3lRuctLNuXesGHD2+7+8d72K7qkUF1dzfr167M+rqmpibq6utwHVOBU7vKicpeXbMptZv+ZyX5qPhIRkYiSgoiIRJQUREQkUnR9CiJSmvbs2UNrayu7d+/O+tjhw4fT3NwcQ1SFrbtyV1ZWMnr0aCoqKvp0TiUFESkIra2tDBs2jOrqaswsq2N37tzJsGHDYoqscHUtt7uzbds2WltbGTNmTJ/OqeYjESkIu3fvZuTIkVknBPmImTFy5Mg+1bZSlBREpGAoIfRff/+GSgoiIhJRn4KIFKTqRatzer6WxecfcPu7775LY2Mj3/nOd7I+9+c//3kaGxs59NBDM9r//vvv55hjjmHcuHFZXytuqilIaWicHfyI9NG7777LLbfc0u22jo6OAx67Zs2ajBMCBEnh+eef79O14qakICICLFq0iJdffpnx48ezcOFCmpqaOOuss5gxY0b0jf6CCy5gwoQJHHvssSxbtiw6trq6mrfffpuWlhZqamq4+OKLOfbYY5kyZQq7du3qdJ0//vGPrFq1ioULFzJ+/Hhefvll6urquPLKK6mtrWXJkiXMmzePlStXRscMHTo0el1fX8/EiRM54YQT+OlPf5rzv4OSgogIsHjxYsaOHcumTZuor68HYOPGjSxZsoQXX3wRgOXLl7NhwwbWr1/P0qVL2bZt237n2bJlC5deeimbN2/m0EMP5d577+20/fTTT2fGjBnU19ezadMmxo4dC8CHH37I+vXrueqqq3qMce3atWzZsoUnn3ySTZs2sWnTJh555JFc/QkA9SmIiPRo0qRJne73X7p0Kffddx8Ar776Klu2bGHkyJGdjhkzZgzjx48HYMKECbS0tGR0rdmze2/+XLt2LWvXruWkk04CYMeOHWzZsoWzzz47o2tkQklBRKQH6cNSNzU1sW7dOh577DGGDBlCXV1dt88DHHzwwdHrgQMH7td8lMm1DjroIPbt2wfAvn37+PDDD4Hg4bQf/OAHfOtb3wLieWhPzUdS3NTBLDkybNgwdu7c2eP27du3c9hhhzFkyBBeeOEFHn/88diuVV1dzYYNGwBYtWoVe/bsAWDq1KksX76c9vZ2ALZu3cqbb77Z5zi6o5qCiBSk3m4hTZeLb8wjR47kjDPO4LjjjmP69Omcf37n60+bNo3bbruNmpoaPvOZz3Dqqaf2+Vpz5szh4osvZunSpZ06lFMuvvhiZs6cyYknnsi0adOiWsSUKVNobm7mtNNOA2Dw4MHcddddHH744X2OpStz95ydLB9qa2tdk+xkruTL3bWWMPduoAzK3YNiLndzczM1NTV9OlZjH3XW3d/SzDa4e21v51TzkYiIRJQUREQkoqQgIiIRJQUREYkoKUjJqV60mmdf257zAdVEyoGSgoiIRPScgogUpiweShzc0QEH9fJxFt6u3JP+DJ190003sWDBAoYMGZL1sYVGNQUREQ48dHZvbrrpJt5///0cR5SMWGsKZjYNWAIMBBrcfXEP+30JWAlMdPfsn0yTZKW+0fXyTSxO65rbAJgf9iNUWxsNFY1A5k/FSnlLHzr7vPPO4/DDD+eee+7hgw8+4Itf/CLXXnst7733Hl/5yldobW1l7969XH311bS1tbF161YmT57MqFGjePjhh5MuSr/ElhTMbCBwM3Ae0Ao8ZWar3P35LvsNA64AnogrFhGR3ixevJjnnnuOTZs2sXbtWlauXMmTTz6JuzNjxgweeeQR3nrrLY444ghWrw6+fGzfvp3hw4dz44038vDDDzNq1KiES9F/cTYfTQJecvdX3P1DYAUws5v9fgL8DNh/uEERkQSkD1F98skn88ILL7BlyxaOP/54HnzwQb7//e/z6KOPMnz48KRDzbk4m4+OBF5NW24FTknfwcxOBo5y99VmtjDGWEREMtZ1iOp0GzduZM2aNfzoRz/inHPO4ZprrkkgwvgkdveRmQ0AbgTmZbDvAmABQFVVFU1NTVlfr729vU/HFbu8lLtyavA7ib9veO1dNcG8tld58Htf5Qh21cyl6XfLYcSn8h9XQor5fT58+PBOw0kPzmKuYnfvdW7jXQcYqjplx44d7Ny5k7POOovrr7+eGTNmMHToULZu3UpFRQUdHR0cdthhzJw5k0GDBnHHHXewc+dODjnkEF5//fVOcynkw969e7sdgnv37t19fh/EmRReA45KWx4drksZBhwHNJkZwCeAVWY2o2tns7svA5ZBMEpqX0aBLObRI/sjL+VuvDX4XZdAR3N47VRH88/3BBXOX5zQxuDmRupqqpKJKyHF/D5vbm7uPOLn1+7teecuMhkltbcxVIcNG8aZZ57JaaedxvTp07nwwguZMmUKEMyRfOedd/LnP/+ZWbNmMWDAACoqKrj11lsZNmwYl1xyCbNmzeKII47Ia0dzT+WurKyMZmfLVpxJ4SngaDMbQ5AM5gBzUxvdfTsQ9cqYWRPwD7r7SESS0tjY2Gn5iiuu6LQ8duxYpk6dut9xl19+OZdffnmsseVLbB3N7t4BXAY8ADQD97j7ZjO7zsxmxHVdERHpu1j7FNx9DbCmy7pue2XcvS7OWEREpHd6ollECkaxzQRZiPr7N1RSEJGCUFlZybZt25QY+sHd2bZtG5WVlX0+hwbEk9JXAMNwSO9Gjx5Na2srb731VtbH7t69u18fhMWqu3JXVlYyevToPp9TSUGKVvWi1TRUtCUdhuRIRUUFY8aM6dOxTU1Nfb4Fs5jFUW41H0n2GmdnNaxxwSjWuEXySElBMqMPVJGyoKQgIiIRJQUREYkoKYiISER3H0lRqQ5nVstUaqA8gHNrqnIdjkjJUU1Bckcd0SJFT0lBREQiaj6SktJQUQ/Aro9GaReRLKimIPEqxOcbCjEmkQKhmoL0W6fO3ATjEJH+U01BilaqqUhEckdJQUREImo+ku4dYLjpVHPR/PCZgYaKDM+Xo6GrVUMQiY9qCiIiElFSEBGRiJqPpGxoyAuR3ikpSE6lj03Usvj8BCMRkb5Q85HkVENFfc8dwXpoTKTgqaYgB9aPD/H0OZTTm2vWXV0HwPw9C1WbECkwqilIYhoq6lVzECkwqilIXqR38opI4VJNQUREIqopSCz01LFIcVJSkIykD22R0bAWcWicHXVci0g81HwkIiIR1RQkK+nNQmoiEik9qimIiEhESUE+oieORcqekoKIiESUFEREJKKkICIiESUFERGJ6JZUKUjV0fzP9czfszB8rQfXROIWa1Iws2nAEmAg0ODui7tsvwS4FNgLtAML3P35OGOSA1vX3BY+tZyfD+DU9SC/k/IkdV2RQhdb85GZDQRuBqYD44C/M7NxXXZrdPfj3X08cANwY1zxiIhI7+LsU5gEvOTur7j7h8AKYGb6Du6+I23xEMBjjEdERHph7vF8DpvZLGCau88Ply8ETnH3y7rsdynwXWAQ8Dl339LNuRYACwCqqqomrFixIut42tvbGTp0aNbHFbusyv3OK+zc3UGLV1Ft+W2/b/Eqjj9yeLT87GvbAai2Nlq8KnqdqX2VIxiw+51erwl0um6x0/u8vGRT7smTJ29w99re9ku8o9ndbwZuNrO5wI+Ai7rZZxmwDKC2ttbr6uqyvk5TUxN9Oa7YZVXuxltZ19zGz/cspKGiMda4uqoB6qwK5t4NwLyoo7mRn0cdzZnHtKtmLoObD7x/6rwtX63LPuACpfd5eYmj3HE2H70GHJW2PDpc15MVwAUxxiMiIr2IMyk8BRxtZmPMbBAwB1iVvoOZHZ22eD6wX9ORSFxSo7xWL1od/YiUu9iaj9y9w8wuAx4guCV1ubtvNrPrgPXuvgq4zMzOBfYAf6WbpiMREcmfWPsU3H0NsKbLumvSXl8R5/VFRCQ7GuZCREQiid99JJKS/pSxiCRDSUEKnqb9FMkfNR+JiEhESUFERCJKCiIiElFSEBGRiJKCiIhElBRkP0ne7dNQUa+7jUQSpKQgIiIRJQUpa6qZiHSmpCDSncbZSUcgkgglBRERiWiYi3Kmb8Mi0oWSghQktfOLJEPNRyIiElFNQSRNakrOhoo2zk04FpEkZJwUzOxE4Kxw8VF3fzqekEREJCkZNR+Z2RXAb4DDw587zezyOAMTEZH8y7Sm8E3gFHd/D8DMfgY8BvwirsBERCT/Mk0KBuxNW94brpMSsK65LekQRKRAZJoU/g/whJndFy5fAPw6npBERCQpGSUFd7/RzJqAM8NVX3f3P8UWlYiIJOKAScHMPubuO8xsBNAS/qS2jXD3d+INTyQ/Ug/Lzd+zMOFIRJLVW02hEfgCsAHwtPUWLn8qprhERCQBB0wK7v6F8PeY/IQjIiJJyvQ5hYcyWSciIsWttz6FSmAIMMrMDuOj21A/BhwZc2wSF42OKiI96K1P4VvAlcARBP0KqaSwA/hljHGJiEgCeutTWAIsMbPL3V1PL0vJ05DdUu4yfU7hF2Z2HDAOqExbf0dcgYmISP5llBTM7B+BOoKksAaYDvwboKQgJSs1jDZAy+LzE4xEJH8ynWRnFnAO8Ia7fx04ERgeW1QiIpKITJPCbnffB3SY2ceAN4Gj4gtLRESS0GvzkZkZ8IyZHQr8b4K7kNoJhs4WEZES0mtScHc3s0nu/i5wm5n9K/Axd38m/vAkLhouu3caD0nKUabNRxvNbCKAu7coIYiIlKZMk8IpwGNm9rKZPWNmz5qZEoOUhYaKej0FLmUj00l2pvbl5GY2DVgCDAQa3H1xl+3fBeYDHcBbwDfc/T/7ci0REem/TB9ey/qD2swGAjcD5wGtwFNmtsrdn0/b7U9Arbu/b2bfBm4A9JVMRCQhmTYf9cUk4CV3f8XdPwRWADPTd3D3h939/XDxcWB0jPGIiEgvzN1736svJzabBUxz9/nh8oXAKe5+WQ/7/5Lg4bjru9m2AFgAUFVVNWHFihVZx9Pe3s7QoUOzPq7YdVvud15h5+6OZALKk32VIxiwO3cTAw6rPAhGFP6cUnqfl5dsyj158uQN7l7b236Z9inEysz+G1ALfLa77e6+DFgGUFtb63V1dVlfo6mpib4cV+y6LXfjrSV/S+qumrkMbm7M2fnqaqpga7gw9+6cnTfX9D4vL3GUO86k8Bqdn3oeHa7rxMzOBX4IfNbdP4gxHpE+S0+i5yYYh0jc4uxTeAo42szGmNkgYA6wKn0HMzsJ+BUww93fjDEWERHJQGxJwd07gMuAB4Bm4B5332xm15nZjHC3emAo8Fsz22Rmq3o4nYiI5EGsfQruvoZgqO30ddekvVZNXESkgMTZfCQiIkVGSUFERCIFcUuq5EdqJrGGitK+HVVE+k41BRERiSgpiIhIREmhXDTOjiaNERHpiZKCiIhE1NEskqXqRaujWte5P2lKNhiRHFNNQUREIkoKZUb9CjnWOFtTdUpJUVIQEZGIkoKIiESUFEREJKKkUOKefW071YtWl/xMayKSG0oKIiISUVIQEZGIkoKIiET0RHOJq7Y2Gioakw5DRIqEagoiWTrgA4B6kE2KnGoKIrmgZCAlQklBJEdSt/3OD2e4a1l8fpLhiPSJmo9ERCSipCASFw2WJ0VISUEkJuua21jX3EZ12JwkUgyUFEREJKKOZpF+0JhSUmqUFEpV1JZ9TqJhiEhxUfORiIhEVFMoQcHE8mGzRk2ysYhIcVFNQUREIkoKIiISUVIQEZGIkoKIiESUFEREJKKkICIiESUFkXzR4HhSBJQURPKgetHqzoPjaQRVKVCxPrxmZtOAJcBAoMHdF3fZfjZwE3ACMMfdV8YZT8mLPmS+lmgY5a7rdJ0HnL5TpMDEVlMws4HAzcB0YBzwd2Y2rstufwHmAZpZXkSkAMRZU5gEvOTurwCY2QpgJvB8agd3bwm37YsxDhERyVCcfQpHAq+mLbeG60TKVkNFvfoSpKAVxYB4ZrYAWABQVVVFU1NT1udob2/v03FFpXIqAFcd38EumwvAvsoR7KqZm2RUiSjkcjdVpv23y/F7size591QuXMnzqTwGnBU2vLocF3W3H0ZsAygtrbW6+rqsj5HU1MTfTmuqDTeCsC8Z79GQ0XQTbOrZi6Dm8uvy6aQy92R9vrcnzTl9Nxl8T7vhsqdO3EmhaeAo81sDEEymAMU5le3EpA+XLbudhGRvoqtT8HdO4DLgAeAZuAed99sZteZ2QwAM5toZq3Al4FfmdnmuOIREZHexdqn4O5rgDVd1l2T9vopgmYlEREpAHqiWUREIkoKIiISKYpbUqUXjbM/mpNZRKQfVFMQEZGIkoKIiETUfFTEUsMwq+lIRHJFSUEkQR8l9vqcP90s0hdqPhIRkYiSgoiIRJQUREQkoqQgIiIRdTQXMY2GKiK5pqQgUiDWXV0HwPw9C2lZfH6ywUjZUlIoMqlbGAEaKhIMRERKkvoUREQkoqQgUmAaKuqhcXbSYUiZUlIQEZGI+hREEqQ7yKTQKCmIFKB1zW0Q3o2kMZEkn9R8JCIiESUFERGJqPmoyKgNWkTipKRQBPTAmojki5JCkVANQUTyQX0KIiISUU1BpMClNx9qoDyJm2oKhUxDHYhInikpFKjqRatZ19zW6VuilKeGinr1KUneKCmIiEhEfQoFTt8QJSV4L6hPQeKlmoJIMWmcrb4miZWSgkgRWdfcpr4miZWSgoiIRNSnUED07U9EkqakUEDUqSyZCqbsvAPm3p10KFJilBQKgToOpQ/WNbcxP6xdtiw+P3gfVU4F6hKNS4qbkkLCqhetpqGiLekwREQAJYVEqO9ARApVrEnBzKYBS4CBQIO7L+6y/WDgDmACsA2Y7e4tccZUKNR/ILmUqnHuqulIOhQpcrElBTMbCNwMnAe0Ak+Z2Sp3fz5tt28Cf3X3T5vZHOBnQMk0sKtGIPmS/iVj3dV1AMzfs7DTPhphVTIRZ01hEvCSu78CYGYrgJlAelKYCfw4fL0S+KWZmbt7jHHlTfp/1NR/UNUQJJcO9H5KbUu994LaROf95+9Z+FGySN3woDuaylqcSeFI4NW05VbglJ72cfcOM9sOjATejjGuPsv2m7+mzpRC0FviqF6Ueh3e8NBNTSOuWkZPc0VoDonkWFxfys1sFjDN3eeHyxcCp7j7ZWn7PBfu0xouvxzu83aXcy0AFoSLnwH+ow8hjaJAk03MVO7yonKXl2zK/Tfu/vHedoqzpvAacFTa8uhwXXf7tJrZQcBwgg7nTtx9GbCsP8GY2Xp3r+3POYqRyl1eVO7yEke54xz76CngaDMbY2aDgDnAqi77rAIuCl/PAn5fKv0JIiLFKLaaQthHcBnwAMEtqcvdfbOZXQesd/dVwK+BfzGzl4B3CBKHiIgkJNbnFNx9DbCmy7pr0l7vBr4cZwxp+tX8VMRU7vKicpeXnJc7to5mEREpPppPQUREIiWZFMxsuZm9Gd7ymlo3wsweNLMt4e/DkowxDj2Uu97MXjCzZ8zsPjM7NMkY49BdudO2XWVmbmajkogtTj2V28wuD//NN5vZDUnFF5ce3ufjzexxM9tkZuvNbFKSMcbBzI4ys4fN7Pnw3/aKcH1OP9tKMikAtwPTuqxbBDzk7kcDD4XLpeZ29i/3g8Bx7n4C8CLwg3wHlQe3s3+5MbOjgCnAX/IdUJ7cTpdym9lkgpECTnT3Y4H/lUBccbud/f+9bwCudffxwDXhcqnpAK5y93HAqcClZjaOHH+2lWRScPdHCO5mSjcT+Ofw9T8DF+Q1qDzortzuvtbdU6OkPU7wvEhJ6eHfG+CfgO8BJdlx1kO5vw0sdvcPwn3ezHtgMeuh3A58LHw9HNia16DywN1fd/eN4eudQDPBqBA5/WwryaTQgyp3fz18/QZQlWQwCfkG8P+SDiIfzGwm8Jq7P510LHl2DHCWmT1hZn8ws4lJB5QnVwL1ZvYqQe2oFGvEETOrBk4CniDHn23llBQi4QNyJfntsSdm9kOC6udvko4lbmY2BPgfBM0I5eYgYARB88JC4B4zs2RDyotvA3/v7kcBf0/wDFRJMrOhwL3Ale6+I31bLj7byikptJnZJwHC3yVXre6Jmc0DvgB8tUyeGB8LjAGeNrMWgiazjWb2iUSjyo9W4HceeBLYRzA+Tqm7CPhd+Pq3BKM0lxwzqyBICL9x91R5c/rZVk5JIX1IjYuA/5tgLHkTTnT0PWCGu7+fdDz54O7Puvvh7l7t7tUEH5Qnu/sbCYeWD/cDkwHM7BhgEOUxUNxW4LPh688BWxKMJRZhje/XQLO735i2Kbefbe5ecj/AXcDrwB6CD4RvEgzJ/RDBm2UdMCLpOPNU7pcIhiffFP7clnSc+Sh3l+0twKik48zTv/cg4E7gOWAj8Lmk48xTuc8ENgBPE7SzT0g6zhjKfSZB09Azaf+fP5/rzzY90SwiIpFyaj4SEZFeKCmIiEhESUFERCJKCiIiElFSEBGRiJKCFBwza+/n8SvN7FNZHnOJmX2tH9e8zczO6OvxcTKzs81so5l1mNmstPUfN7N/TTI2KTxKClJSzOxYYKC7v5LFMQe5+23ufkc/Ln0qwYCDGV2vH9fpeq46M7u9l93+AswDGtNXuvtbwOuFmswkGUoKUrAsUG9mz5nZs2Y2O1w/wMxuCecMeNDM1qR9A/4qaU90mlm7mf1TOP78Q2b28XB9k5ndZGbrgSvM7Mdm9g/htk+b2Tozezr8hj02XL/QzJ4K56a4Nu0aNcCL7r7XzC4O93nazO4Nx2HCzG4PaxNPADeY2SHhvABPmtmfwgH8MLNqM3s0vO5GMzu9v39Hd29x92cIhrzo6v7wbyYCKClIYftbYDxwInAuwSiYnwzXVwPjgAuB09KOOYPgydaUQ4D1Hswt8AfgH9O2DXL3Wnf/eZfr/ga42d1PBE4n+DY9BTiaYEyd8cAEMzs73H86kGqG+Z27TwyPbSZ42jZlNHC6u38X+CHwe3efRDAsRb2ZHUIwbs157n4yMBtYmtmfqs/WA2fFfA0pIjmrxorE4EzgLnffSzDo1x+AieH637r7PuANM3s47ZhPAm+lLe8D7g5f38lHg6aRtj5iZsOAI939PgB33x2un0IwYc+fwl2HEiSJR4CpwNfD9ceZ2fXAoeE+D6Sd/rdhWQjPNSNVOwEqgf9CMIbPL81sPLCXYCjs/YQ1joPDa4wws03hpu+7+wPdHdODN4EjsthfSpySgpSaXQQfsD1JH9flvSzOa8D/dPdfdVoZNA8d6u6pSV1uBy5w96fD0WnrerieAV9y9//ocr4fA20EtaMBwO5uC+F+Srh/HTDP3edlUZZ0lQR/MxFAzUdS2B4FZpvZwLAv4GzgSeDfgS+FfQtVdP7gbQY+nbY8AEj1N8wF/u1AF/RgRqtWM7sAwMwODj/4HwC+EY5lj5kdaWaHEzT9pNdUhhE0N1Vw4Lb6B4DLU3MdmNlJ4frhwOthLehCYOCB4s2BYwgGzxMBlBSksN1HMCLk08Dvge95MPz1vQSjYz5P0CS0EdgeHrOa/b+dT7JgkvfPAddlcN0Lgf9uZs8AfwQ+4e5rCe7eeczMngVWEiSA9P4EgKsJRun8d+CFA1zjJ0AF8IyZbQ6XAW4BLjKzp4H/Sna1mW6Z2UQzawW+DPwqvF7KZIK/mQiARkmV4mRmQ9293cxGEtQeznD3N8xsMME39zPCu4Ha3X1ojHFsBE5x9z1xXSNOZvYIMNPd/5p0LFIYlBSkKJlZE0Fn7iDgBne/PW3bVIKJSP4Sd1IoZmGT3Bnufn/SsUjhUFIQEZGI+hRERCSipCAiIhElBRERiSgpiIhIRElBREQiSgoiIhL5/3/wXzNBJXAYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHF1JREFUeJzt3X98VuV9//HXBwxGBEGhZlXYglb7Df5CidjW6m6mYqx9gP3OCkNt3Qq0q/jVzaal3/mjuj9Gm9UJm9WylDlnM7RYHY/BJuJIdd+qEChWIVYoj7QGW6w4KKFEiHy+f9wnh5OQO7mT3Oc+933n/Xw8fHh+XOecz5WE+3Nf13XOdczdERERARiWdAAiIlI4lBRERCSkpCAiIiElBRERCSkpiIhISElBRERCSgoiIhJSUhARkZCSgoiIhI5LOoD+Gj9+vFdWVmbcf+DAAU488cT8BZQw1bf0DbU6q77x2LRp07vu/qG+yhVdUqisrKSpqSnj/sbGRlKpVP4CSpjqW/qGWp1V33iY2S+yKafuIxERCSkpiIhISElBRERCRTemICKl6fDhw7S2tjJmzBiam5uTDidvcl3f8vJyJkyYQFlZ2YCOV1IQkYLQ2trK6NGjGTduHCeddFLS4eTN/v37GT16dE7O5e7s2bOH1tZWJk2aNKBzqPtIRApCe3s748aNw8ySDqVomRnjxo2jvb19wOdQUhCRgqGEMHiD/RkqKYiISEhjCiJSkCoXrc7p+VoWX9vr/r1799LQ0MCXv/zlfp/7U5/6FA0NDYwdOzar8s888wxnn302kydP7ve14qaWgiSictHq8D+RQrB3716+853v9Livo6Oj12PXrFmTdUKAdFLYtm3bgK4VNyUFERFg0aJF/PznP2fKlCnU1tbS2NjIZZddxsyZM8Nv9Ndddx1Tp07lnHPOYdmyZeGxlZWVvPvuu7S0tFBVVcX8+fM555xzmDFjBgcPHuxynR//+MesWrWK2tpapkyZws6dO0mlUtxxxx1UV1ezZMkSbrnlFlauXBkeM2rUqHC5rq6Oiy++mPPPP59777035z8HJQUREWDx4sWceeaZbNmyhbq6OgA2b97MkiVLePPNNwFYvnw5mzZtoqmpiaVLl7Jnz55jzrN9+3ZuvfVWtm7dytixY3nqqae67P/EJz7BzJkzqaurY8uWLZxxxhkAHDp0iKamJu68886MMa5du5bt27ezYcMGtmzZwqZNm3jhhRdy9SMANKYgIpLRtGnTutzvv3TpUp5++mkA3nrrLbZv3864ceO6HDNp0iSmTJkCwNSpU2lpacnqWrNnz+6zzNq1a1m7di0XXnghAG1tbWzfvp3LL788q2tkQ0lBRCSD6JTWjY2NrFu3jpdeeomRI0eSSqV6fB7g+OOPD5eHDx9+TPdRNtc67rjjOHLkCABHjhzh0KFDQPrhtK9//et88YtfHFB9sqHuIxERYPTo0ezfvz/j/n379nHyySczcuRI3njjDV5++eXYrlVZWcmmTZsAWLVqFYcPHwbg6quvZvny5bS1tQGwa9cu3nnnnQHH0RO1FESkIPV1C2mujRs3jksvvZRzzz2Xa665hmuv7Xr9mpoaHnnkEaqqqvjoRz/Kxz72sQFfa86cOcyfP5+lS5fy6KOPHrN//vz5zJo1iwsuuICampqwFTFjxgyam5v5+Mc/DqQHoB9//HFOPfXUAcfSnbl7zk6WD9XV1a6X7BxVrPWN3oran3/8xVrfwRgqdW5ubqaqqiqncwEVgzjq2/mzjDKzTe5e3dex6j4SEZGQkoKIiISUFEREJKSkICIiISUFEREJKSmIiEhIzymISGFq6Hvah36Z+0SvuwczdfaDDz7IggULGDly5ECjKxhqKYiI0PvU2X158MEH+d3vfpfjiJKhloKICF2nzr7qqqs49dRTefLJJ3n//ff5zGc+w3333ceBAwe44YYbaG1t5YMPPuDuu+9m9+7dvP3220yfPp3x48ezfv36pKsyKEoKIiKkp85+/fXX2bJlC2vXrmXlypVs2LABd2fmzJm88MIL/OY3v+G0005j9er0E/n79u1jzJgxPPDAA6xfv57x48cnXIvBU/eRiEg30SmqL7roIt544w22b9/Oeeedx3PPPcfXvvY1XnzxRcaMGZN0qDmnloKISDe9TVG9efNm1qxZw1133cUVV1zBPffck0CE8VFLQUSErtNZZ5qi+u2332bkyJHcdNNN1NbWsnnz5mOOLXZqKYhIYerjFtJc6z519ty5c4+ZonrHjh3U1tYybNgwysrKePjhhwFYsGABNTU1nHbaaRpoFhEpFQ0NDV3Wb7/99i7rZ555JldfffUxx912223cdtttscaWL+o+EhGRkJKCiIiEYk0KZlZjZj8zsx1mtqiXcn9sZm5mfb4VSERKV7G9CbIQDfZnGNuYgpkNBx4CrgJagY1mtsrdt3UrNxq4HXglrlikwDTMpr5sNwDzDtcmHIwUivLycvbs2cOIESOSDqVouTt79uyhvLx8wOeIc6B5GrDD3XcCmNkKYBawrVu5vwa+CejTQWQImzBhAq2trezdu3dQH2rFpr29Paf1LS8vZ8KECQM+Ps6kcDrwVmS9FbgkWsDMLgImuvtqM1NSEBnCysrKmDRpEo2NjVx44YVJh5M3hVbfxG5JNbNhwAPALVmUXQAsAKioqKCxsTFj2ba2tl73l5qirG/51Rys6gDgTu/oV/xFWd9BGmp1Vn2TFWdS2AVMjKxPCLZ1Gg2cCzSaGcDvAavMbKa7N0VP5O7LgGUA1dXVnkqlMl60sbGR3vaXmqKsb8PDrGtOjyl8+3AtLTemsj60KOs7SEOtzqpvsuJMChuBs8xsEulkMAeY27nT3fcB4ZSCZtYIfKV7QpASkesXpohILGJLCu7eYWYLgWeB4cByd99qZvcDTe6+Kq5rSxGKJo08T28gIkfFOqbg7muANd229TiloLun4oxFRET6pieaRUQkpKQgIiIhJQUREQkpKYiISEhJQUREQnrJjuRE5aLV4XLL4msTjEREBkNJQWLVmSzqy3ZzZVVFwtGISF/UfSQiIiElBRERCan7SBJXuWh1+NIdgHlBl5PGJkTyT0lBci466CwixUXdRyIiElJSEBGRkLqPJDb1ZXVJhyAi/aSkIIlS4hApLOo+kpyoL6vTB7xICVBSEBGRkLqPJKfUWhApbmopiIhISElBRERC6j6SftEU2SKlTS0FEREJqaUgfWuYHVn5XGJhiEj8lBSkT+uad/ddKGad3VZ3ntdBKtlQREqakoIUnPC21obHYO4TyQYjMsRoTEFEREJKCiIiElJSEBGRkMYUJG8KYcBaRHqnloKIiISUFEREJKSkICIiIY0pyMB1edJZREqBkoL0S9f3JVQkFoeIxEPdRyIiElJSEBGRUKzdR2ZWAywBhgP17r642/4vAbcCHwBtwAJ33xZnTJKZ3pUgIrG1FMxsOPAQcA0wGfgTM5vcrViDu5/n7lOAbwEPxBWPiIj0Lc6WwjRgh7vvBDCzFcAsIGwJuPtvI+VPBDzGeCQLRweSC6SlENzhVF+2m3mHaxMORqT0mXs8n8Nmdj1Q4+7zgvWbgUvcfWG3crcCfwmMAP7I3bf3cK4FwAKAioqKqStWrMh43ba2NkaNGpWzehS6XNb3tV37qLT0VBSjTzs73L7/7Td7LD+6/Oh3iv3tHTmJoa/zHyk/hTGnjM/5tQqZ/qZLW77qO3369E3uXt1XucRvSXX3h4CHzGwucBfw+R7KLAOWAVRXV3sqlcp4vsbGRnrbX2pyWd9bFq2mvqwBgNTcxnD7uru/0WP5aBo4IScRdJWqOnrLa+e8SQer5pJKXR9uHwrjIPqbLm2FVt84k8IuYGJkfUKwLZMVwMMxxiMlIpoIuj43UZpJQSSf4rwldSNwlplNMrMRwBxgVbSAmZ0VWb0WOKbrSERE8ie2loK7d5jZQuBZ0rekLnf3rWZ2P9Dk7quAhWZ2JXAY+B966DoSEZH8iXVMwd3XAGu6bbsnsnx7nNcXEZH+0RPNIiISSvzuIylQmgFVZEhSS0FEREJKCiIiElL3kRSdrs8miEguqaUgIiIhJQUREQkpKYiISEhJQUREQkoKIiISyvruIzO7ALgsWH3R3V+NJyQREUlKVknBzG4H5gM/DDY9bmbL3P3vY4tMhrzOdyiISP5k21L4Aum3ph0AMLNvAi8BSgoiIiUk2zEFAz6IrH8QbBMRkRKSbUvhn4BXzOzpYP064HvxhCQiIknJKim4+wNm1gh8Mtj0p+7+k9iikvwKZkStL1MfvshQ12tSMLOT3P23ZnYK0BL817nvFHd/L97wREQkn/pqKTQAnwY2AR7ZbsH6GTHFJSIiCeg1Kbj7p4P/T8pPOCIikqRsn1N43t2v6GubSKKib4ub+0RycYgUsb7GFMqBkcB4MzuZo7ehngScHnNskmv60BSRPvTVUvgicAdwGulxhc6k8FvgH2KMS/KkctFq3XUkIqG+xhSWAEvM7DZNaSEiUvqyfU7h783sXGAyUB7Z/lhcgYmISP5lO9B8L5AinRTWANcA/w0oKUjBiE6gd2WCcYgUs2ynubgeuAD4ibv/qZlVAI/HF5bEaV3zbuYtWp10GCJSgLKdEK/d3Y8AHWZ2EvAOMDG+sEREJAl9thTMzICfmtlY4B9J34XURnrqbJGCVBlpCbUsvjbBSESKS59Jwd3dzKa5+17gETP7T+Akd/9p/OFJTkSfTxAR6UW23UebzexiAHdvUUIQESlN2Q40XwLcaGa/AA4QTIjn7ufHFplIrnS2lPQUt0ifsk0KV8cahYiIFIRsH177RdyBSP90DqTeeV4HqQEcX19Wl9N4Ck3X+lUkFodIscm2pSDFRoPLIjIASgolprMFUV+2myur9A25J7pdVSSzbO8+GhAzqzGzn5nZDjNb1MP+vzSzbWb2UzN73sz+IM54RESkd7ElBTMbDjxEep6kycCfmNnkbsV+AlQHdzGtBL4VVzwiItK3OLuPpgE73H0ngJmtAGYB2zoLuPv6SPmXgZtijKf0NczO+G6E6GRxIiKZxNl9dDrwVmS9ld7f1vYF4D9ijEdERPpg7h7Pic2uB2rcfV6wfjNwibsv7KHsTcBC4A/d/f0e9i8AFgBUVFRMXbFiRcbrtrW1MWrUqNxUooC9tmsfABUnwKmnjElvfG8n+9s7cnL+0eVHG5G5OmcuHCk/hWHt7w36PC2eHoQ/7/Qxgz5X3IbK33Qn1Tce06dP3+Tu1X2Vi7P7aBddZ1KdEGzrwsyuBP6KDAkBwN2XAcsAqqurPZVKZbxoY2Mjve0vFbdEnlO4obO+DQ/nrJsoFblzqZC6ng5WzeWE5oZBn+fbh2sBaLkxNehzxW2o/E13Un2TFWdS2AicZWaTSCeDOcDcaAEzuxD4LukWxTsxxiL9VEiJQETyJ7ak4O4dZrYQeBYYDix3961mdj/Q5O6rgDpgFPCD9Azd/NLdZ8YVUymqtN16UE1EcibWh9fcfQ3p13dGt90TWdZbE0VECkisD6+JiEhxUVIQEZGQkoKIiISUFEREJKSkICIiISUFEREJKSmIiEhISUFEREJKCiIiElJSEBGRkN7RLENSfVldeqHhMZj7RM+FOueUyrRfpAQpKRSjyBvWDnadeFZEZFDUfSQiIiElBRERCan7SCRQGbzNDqBl8bUJRiKSHLUUREQkpJaCDGnrmnfD3SkA6stgXvDuZpGhSklBpAeVi1aHd3jp9YAylKj7SEREQmopFJPOh6lERGKiloKIiISUFEREJKTuo0KnLqO8CudEykDPMkipU0tBRERCSgoiIhJSUhARkZCSgoiIhDTQXCA0gCkihUAtBRERCamlUODWNe8Ol6+squizjMSgy23Bn0ssDJF8UEtBRERCaikUEbUIRCRuaimIiEhILQWRgYqONcx9Irk4RHJISUGkH7rOjdTzwL9IMVNSEMkBPWcipSLWpGBmNcASYDhQ7+6Lu+2/HHgQOB+Y4+4r44ynaGhmVBFJSGxJwcyGAw8BVwGtwEYzW+Xu2yLFfgncAnwlrjhE4qK7waQUxdlSmAbscPedAGa2ApgFhEnB3VuCfUdijENkUPThL0OJuXs8Jza7Hqhx93nB+s3AJe6+sIeyjwL/nqn7yMwWAAsAKioqpq5YsSLjddva2hg1atTgK5Bnr+3aFy6fd8KecHl/e0evxx0pP4Vh7e/FFlehKdT6tvjRQefzTh+T03MX69/0QKm+8Zg+ffomd6/uq1xRDDS7+zJgGUB1dbWnUqmMZRsbG+ltf6G6JTJQWV/2WLh8Qh/HHayaywnNDTFFVXgKtb7fPlwbLrfcmMrpuYv1b3qgVN9kxfnw2i5gYmR9QrBNREQKVJxJYSNwlplNMrMRwBxgVYzXExGRQYotKbh7B7AQeBZoBp50961mdr+ZzQQws4vNrBX4LPBdM9saVzwicaovq+v2YJtIcYp1TMHd1wBrum27J7K8kXS3koiIFICiGGgWKQV66lmKgZKCSI7pw1+KmZJCgVB/dGmI/h7nRW5TFSkWSgp5pm+RQ0v09y1SDJQUkqSJ70SkwCgpiMREXUlSjPQ6TpE8qy+rS7cS1VKUAqSkICIiIXUfieSB7i6TYqGkkC9BV0F92W71L4tIwVL3kYiIhJQUREQkpO6jPKhctJr6Mr3SUUQKn1oKIiISUkshAUfvRKnotZyUrnXN6ZbjvEWrNd2JFBS1FEREJKSWgkjCNEmiFBK1FEREJKSkIJIgvdtZCo26j0QKiLqSJGlKCgnqvANFRNNsS6FQ95GIiITUUsghNf0l5xpmQ/nV0PAwzH0i6WhkCFBSiFNkZlSR/uqcHuVgVUfSocgQoqQgUmB6uhtpXfNu5gUtUbVCJU4aUxARkZBaCjHRzKgiUoyUFAagtwFlPYgkcQn/thoe06CzxEZJQaQYBTcxRJOD7n6TXFBSyFL0H5xIoVh3dyqypofeZPCUFPpJT55K0jI9Cd+161ItBRkYJQWREqSuJBkoJYVBUreSiJQSJYVB0J1GUgw6xx3mHa7tudXQOWgNuqtJlBSy0jBbzxxIUcnmC0tnK7e+bDdXVvXwvvAe7nCS0hdrUjCzGmAJMByod/fF3fYfDzwGTAX2ALPdvSXOmPoU+dakqa2llNSX1bHu7rrI+tF94d96l7uZ6DlZ9EatjqIXW1Iws+HAQ8BVQCuw0cxWufu2SLEvAP/j7h8xsznAN4HZx54tf5QIRI7KNOdSprG0lvPzEpbEKM6WwjRgh7vvBDCzFcAsIJoUZgHfCJZXAv9gZubuHmNcx4j+gUe/PYnI0a6oTK2MqHXNR5fnZXETRue58zLeoVZMVuJMCqcDb0XWW4FLMpVx9w4z2weMA96NJaIMXUNKBCK5158bMaJdWwer5rLu7m+E+6JdWN1bKD1d45guryABdOkFiHSTZewi6ytxlGiSsbi+lJvZ9UCNu88L1m8GLnH3hZEyrwdlWoP1nwdl3u12rgXAgmD1o8DPern0eOJKKoVJ9S19Q63Oqm88/sDdP9RXoThbCruAiZH1CcG2nsq0mtlxwBjSA85duPsyYFk2FzWzJnevHlDERUj1LX1Drc6qb7LifJ/CRuAsM5tkZiOAOcCqbmVWAZ8Plq8H/ivf4wkiInJUbC2FYIxgIfAs6VtSl7v7VjO7H2hy91XA94B/MbMdwHukE4eIiCQk1ucU3H0NsKbbtnsiy+3AZ3N82ay6mUqI6lv6hlqdVd8ExTbQLCIixUfvaBYRkVBRJwUzW25m7wS3tnZuqzOzN8zsp2b2tJmNTTLGXOqpvpF9d5qZm9n4JGKLQ6b6mtltwe94q5l9K6n4ci3D3/MUM3vZzLaYWZOZTUsyxlwys4lmtt7MtgW/y9uD7aeY2XNmtj34/8lJx5oLvdS3oD6zijopAI8CNd22PQec6+7nA28CX893UDF6lGPri5lNBGYAv8x3QDF7lG71NbPppJ+Ev8DdzwH+NoG44vIox/5+vwXc5+5TgHuC9VLRAdzp7pOBjwG3mtlkYBHwvLufBTwfrJeCTPUtqM+sok4K7v4C6buWotvWuntHsPoy6ecjSkJP9Q38HfBVoKQGiDLU98+Bxe7+flDmnbwHFpMM9XXgpGB5DPB2XoOKkbv/yt03B8v7gWbSsxzMAv45KPbPwHXJRJhbmepbaJ9ZRZ0UsvBnwH8kHUSczGwWsMvdX006ljw5G7jMzF4xsx+Z2cVJBxSzO4A6M3uLdKuolFq+ITOrBC4EXgEq3P1Xwa5fA/2cqrXwdatvVOKfWSWbFMzsr0g3176fdCxxMbORwP8l3a0wVBwHnEK6+V0LPGlmlmxIsfpz4C/cfSLwF6Sf7SkpZjYKeAq4w91/G90XPMxaUi3gTPUtlM+skkwKZnYL8GngxhJ/QvpMYBLwqpm1kG52bjaz30s0qni1Aj/0tA3AEdJzx5SqzwM/DJZ/QHr24ZJhZmWkPyC/7+6d9dxtZh8O9n8YKJkuwgz1LajPrJJLCsGLfb4KzHT33yUdT5zc/TV3P9XdK929kvQH5kXu/uuEQ4vTM8B0ADM7GxhBaU+e9jbwh8HyHwHbE4wlp4IW3veAZnd/ILIrOv3N54F/y3dscchU30L7zCrqh9fM7F+BFOlviruBe0n3uR7P0Yn1Xnb3LyUSYI71VF93/15kfwtQ3X2W2WKV4ff7L8ByYApwCPiKu/9XUjHmUob6/oz02wuPA9qBL7v7pqRizCUz+yTwIvAa6RYfpLtDXwGeBH4f+AVwg7v3dINFUemlvkspoM+sok4KIiKSWyXXfSQiIgOnpCAiIiElBRERCSkpiIhISElBRERCSgpSEsysbZDHrzSzM/p5zJfM7HODuW5/mdn/MrOXzOx9M/tKZPsIM3sheNe5yIApKciQZ2bnAMPdfWc/jjnO3R9x98dyGEelmTX2Uew94P/QbXZYdz9EekbR2bmKR4YmJQUpKZZWZ2avm9lrZjY72D7MzL4TzFv/nJmtMbPrg8NuJPLUrJm1mdnfBXPeP29mHwq2N5rZg2bWBNxuZt/o/LZuZh8xs3Vm9qqZbTazM4PttWa2MZgr/77B1s/d33H3jcDhHnY/E9RFZMCUFKTU/G/STztfAFxJeobRDwfbK4HJwM3AxyPHXApEnxI+EWgK3tfwI9JPFnca4e7V7v7tbtf9PvCQu18AfAL4lZnNAM4iPV/RFGCqmV2ek1r27HWg1GeNlZip/1FKzSeBf3X3D0hPrPYj0h+UnwR+4O5HgF+b2frIMR8GfhNZPwI8ESw/ztEJ6YhsD5nZaNLz4j8N4O7twfYZpF9+9JOg6CjSSeKFbsc/TXpiwxHA75vZlmDXEnf/p2wr7u4fmNkhMxsdzNcv0m9KCiJwECjvZX90LpgD/TivAX/j7t/trZC7fwbCOfYfdfdUP67R3fGk50gSGRB1H0mpeRGYbWbDg7GAy4ENwP8D/jgYW6ggPfFcp2bgI5H1YUDneMNc4L97u2DwrbzVzK4DMLPjg3ddPAv8WTB/PmZ2upmdOtgKZmJm44B33b2n8QaRrKilIKXmadLjBa+S/ob/VXf/tZk9BVwBbAPeAjYD+4JjVpNOEuuC9QPANDO7i/Rc/tnc0XMz8F0zu5/0IPBn3X2tmVUBLwXvAWoDbmIQ7wcI3pXRRPoVnUfM7A5gcvCylulBXUQGTLOkypBhZqPcvS34Rr0BuDRIGCcA64P1D8yszd1HJRtt/5nZD4FF7v5m0rFI8VJLQYaSfzezsaQHdP+682VE7n7QzO4l/dL4XyYZ4ECZ2QjgGSUEGSy1FEREJKSBZhERCSkpiIhISElBRERCSgoiIhJSUhARkZCSgoiIhP4/FgVTsgoqg6gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(cv['log_parea_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price'] / test['building_area']), bins=100, label='test',\n",
    "         normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price/area + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()\n",
    "\n",
    "plt.hist(cv['log_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price']), bins=100, label='test', normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "plt.hist(np.log1p(cv['building_area']), bins=100, label='train', normed=True)\n",
    "plt.hist(np.log1p(test['building_area']), bins=100, label='test', normed=True, alpha=0.7)\n",
    "plt.xlabel('log(building_area + 1)'); plt.ylabel('ratio'); plt.yscale('log')\n",
    "plt.legend(); plt.grid(); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.382257e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.926757e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.113643e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>6.038317e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.064189e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.235242e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.741861e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.095633e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.973214e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.112596e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.382257e+07\n",
       "1     BTshNOJyKHnT2YIT  3.926757e+06\n",
       "2     dhdymr0lV8N5kZOT  1.113643e+07\n",
       "3     VEwyGGMcD56w5BOc  6.038317e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.064189e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.235242e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.741861e+07\n",
       "9997  YunNwAhcqkf6YclI  1.095633e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.973214e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.112596e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "test[['building_id', 'total_price']].check()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output file to: output/stack_meanlog_11_1-11,14-17.csv\n"
     ]
    }
   ],
   "source": [
    "output_filename = 'output/stack_meanlog_{}_{}.csv'.format(stack_idx, models)\n",
    "print('Output file to:', output_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[['building_id', 'total_price']].to_csv(output_filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.382257e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.926757e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.113643e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>6.038317e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.064189e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.235242e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.741861e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.095633e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.973214e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.112596e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.382257e+07\n",
       "1     BTshNOJyKHnT2YIT  3.926757e+06\n",
       "2     dhdymr0lV8N5kZOT  1.113643e+07\n",
       "3     VEwyGGMcD56w5BOc  6.038317e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.064189e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.235242e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.741861e+07\n",
       "9997  YunNwAhcqkf6YclI  1.095633e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.973214e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.112596e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "pd.read_csv(output_filename).check()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission for linear mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict for liear mean with [3, 4, 5, 7, 8, 10, 15, 16, 17]\n"
     ]
    }
   ],
   "source": [
    "print('Predict for liear mean with', best_models_subset_none)\n",
    "test['total_price'] = predict_mean_subset(test, best_models_subset_none, is_per_area=False, is_log=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6571: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHTNJREFUeJzt3Xt8VeWd7/HPDwgGBEGgpiqeCbXaA95QAnipTmjlYu0B29ri0Kq0BTqtdHTqMMVpddSevoaaUxVaraWph+lpI1qsDq+BGTDWVGfqhUvxgrGivGgNaFCsSJQol9/5Y60sd0JC9k722mtfvu/XKy/2Wntdfk+yWb/9PM96nmXujoiICECfpAMQEZH8oaQgIiIRJQUREYkoKYiISERJQUREIkoKIiISUVIQEZGIkoKIiESUFEREJNIv6QAyNWLECK+srMx4v3feeYcjjzwy+wHlOZW7tKjcpSWTcm/YsOENd/9Qd9sVXFKorKxk/fr1Ge/X0NBAdXV19gPKcyp3aVG5S0sm5TazP6WznZqPREQkoqQgIiIRJQUREYkUXJ+CiBSnffv20dTURGtra8b7DhkyhMbGxhiiym+dlbu8vJyRI0dSVlbWo2MqKYhIXmhqamLw4MFUVlZiZhntu2fPHgYPHhxTZPmrY7ndnV27dtHU1MSoUaN6dEw1H4lIXmhtbWX48OEZJwT5gJkxfPjwHtW22igpiEjeUELovd7+DpUUREQkoj4FEclLlQtXZfV42xZdfNj333rrLerq6vjGN76R8bE/9alPUVdXx9ChQ9Pa/sEHH+Tkk09mzJgxGZ8rbqopSOGrm5l0BFIE3nrrLe68885O39u/f/9h9129enXaCQGCpPD888/36FxxU1IQEQEWLlzIyy+/zNixY1mwYAENDQ2cf/75TJ8+PfpGf8kllzBu3DhOOeUUli5dGu1bWVnJG2+8wbZt2xg9ejRz587llFNOYcqUKezdu7fdeX7/+9+zcuVKFixYwNixY3n55Zeprq7mmmuuoaqqisWLFzN79mxWrFgR7TNo0KDodU1NDePHj+f000/n+9//ftZ/D0oKIiLAokWLOPHEE9m0aRM1NTUAbNy4kcWLF/Piiy8CcPfdd7NhwwbWr1/PkiVL2LVr1yHH2bJlC1dddRWbN29m6NCh3H///e3eP/fcc5k+fTo1NTVs2rSJE088EYD333+f9evXc+2113YZ49q1a9myZQtPPfUUmzZtYtOmTTz66KPZ+hUA6lMQEenShAkT2t3vv2TJEh544AEAXnnlFbZs2cLw4cPb7TNq1CjGjh0LwLhx49i2bVta55o5s/tm0LVr17J27VrOPPNMAN5++222bNnCBRdckNY50qGkICLShdRpqRsaGqivr+fxxx9n4MCBVFdXdzoe4Igjjohe9+3b95Dmo3TO1a9fPw4ePAjAwYMHef/994FgcNp1113H1772NSCeQXtqPhIRAQYPHsyePXu6fH/37t0cffTRDBw4kBdeeIEnnngitnNVVlayYcMGAFauXMm+ffsAmDp1KnfffTctLS0A7Nixg507d/Y4js6opiAieam7W0hTZeMb8/DhwznvvPM49dRTueiii7j44vbnnzZtGnfddRejR4/mYx/7GGeffXaPz3XZZZcxd+5clixZ0q5Duc3cuXOZMWMGZ5xxBtOmTYtqEVOmTKGxsZFzzjkHgAEDBnDPPfdwzDHH9DiWjszds3awXKiqqnI9ZCd9JVHuupkw6952q0qi3J0o5HI3NjYyevToHu2ruY/a6+x3aWYb3L2qu2Oq+UhERCJKClIc6mZqEJtIFigpiIhIRElBREQiSgoiIhJRUpCiU7lwFc9u3531WTZFSoHGKYhIfsrgxoEB+/dDv24uZx1uW+6oN1Nn33777cybN4+BAwdmvG++UU1BRITDT53dndtvv5133303yxElQzUFERHaT509efJkjjnmGO677z7ee+89PvOZz3DTTTfxzjvv8IUvfIGmpiYOHDjA9ddfT3NzMzt27GDSpEmMGDGCRx55JOmi9EqsScHMpgGLgb5Arbsv6mK7zwErgPHunvlwZSl59Y3NAMxRP4L00KJFi3juuefYtGkTa9euZcWKFTz11FO4O9OnT+fRRx/l9ddf57jjjmPVquBztnv3boYMGcKtt97KI488wogRIxIuRe/F1nxkZn2BO4CLgDHA35jZIc+eM7PBwNXAk3HFIqWn0pqpLatJOgwpUKlTVJ911lm88MILbNmyhdNOO42HHnqIb3/72zz22GMMGTIk6VCzLs6awgTgJXffCmBmy4EZQMdn0H0P+AGwIMZYRETS1nGK6lQbN25k9erVfPe73+WTn/wkN9xwQwIRxifOjubjgVdSlpvCdREzOws4wd1V55f4aAoMSUPqdNYdp6jevn07O3fuZMeOHQwcOJAvfelLLFiwgI0bNx6yb6FLrKPZzPoAtwKz09h2HjAPoKKigoaGhozP19LS0qP9Cl1JlLt8KntHBw87v9aDfw+WD2Pv6FlB2cunBtsV+++Bwv57DxkypP2F9X/Vpr3vgQMH6Nu37+E36uai3b9/fyZMmMCYMWOYPHkyn/3sZ5k4cSIQPADnZz/7GVu3buX666+nT58+9OvXj9tuu409e/ZwxRVXMGXKFI499tiovyEXDhw40Gkyam1t7fHnILaps83sHOBGd58aLl8H4O7/Ei4PAV4GWsJdPgy8CUw/XGezps7OTEmUu27mBx3N+4JWyB+dvp0BjXVc+L2GD2oJ3dynXgwK+e+tqbMzV2hTZ68DTjKzUWbWH7gMWNn2prvvdvcR7l7p7pXAE3STEEREJF6xJQV33w/MB9YAjcB97r7ZzG42s+lxnVekHfUliGQk1j4Fd18NrO6wrtOuenevjjMWkVJqRipU7o6ZJR1GQettl4CmuRCRvFBeXs6uXbt6fVErZe7Orl27KC8v7/ExNM2FiOSFkSNH0tTUxOuvv57xvq2trb26EBaqzspdXl7OyJEje3xMJQXpPTXLSBaUlZUxatSoHu3b0NDAmWeemeWI8l8c5VZSkILV9ryE2rLmhCMRKR7qU5D0aFSwSElQUhARkYiSgoiIRJQUREQkoqQgPaP+BZGipLuPJF5Zvl21MsMnq7VNlAdw4eiKrMQgUsxUUxARkYiSgoiIRJQUREQkoqQguRVzB3VtWQ21ZTWxnkOkmCkpSK/VNzZT39iccSewiOQfJQUpOKoNiMRHSUG6lmFTT21ZTU7HLyg5iGSfxilIxioXrqK2rJk50SylCQckIlmjpCA9pm/pIsVHzUdSMtQhLtI91RQkHpobSaQgqaYgWdX2TTx1zqGsqZupJiuRmCkpSNbpwi1SuNR8JLFpN0NpyvrUNv1tiy7OYUQi0h3VFCQzPW3C6eQZz7ke15D0eUUKgWoKkhOx9DGISNappiAiIhElBRERiaj5SNKi5h+R0qCkIDmnW1ZF8peSghye7tIRKSnqUxARkYiSgoiIRJQUREQkoqQgIiIRJQUREYkoKYiISES3pEpe6uzpaLVlGkAnErdYk4KZTQMWA32BWndf1OH9vwWuAg4ALcA8d38+zpjk8FIvxrm4CNc3NjMnPGcup9FO6rwi+S625iMz6wvcAVwEjAH+xszGdNiszt1Pc/exwC3ArXHFIwWgk4FytWU1GgEtkkNx9ilMAF5y963u/j6wHJiRuoG7v52yeCTgMcYjIiLdMPd4rsNmdikwzd3nhMuXAxPdfX6H7a4CvgX0Bz7h7ls6OdY8YB5ARUXFuOXLl2ccT0tLC4MGDcp4v0KXabmf3b47el1puWnD3+YVAJw2YBcM+0i7ONpi2OYVGcVzsHwYfVrfTO+8xw/JOOZ8pc95acmk3JMmTdrg7lXdbZd4R7O73wHcYWazgO8CV3ayzVJgKUBVVZVXV1dnfJ6GhgZ6sl+hy7Tcs9v1KdTFENGhfrhvAQDbTl8D1fe2i6Mthh/uW5BRPHtHz2JA4+G3j877xepMQ85b+pyXljjKHWdS2A6ckLI8MlzXleXAT2KMR6SdD/oq1NEs0ibOpLAOOMnMRhEkg8uAWakbmNlJKc1FFwOHNB1J7pVax27qHVe6E0lKXWxJwd33m9l8YA3BLal3u/tmM7sZWO/uK4H5ZnYhsA/4C500HUkO1c3M27EApZaoRJISa5+Cu68GVndYd0PK66vjPL+kSc9MEJGQprkQEZFI4ncfibQ1DdU3Eo0yFpFkqKYgIiIRJQUREYkoKYiISERJQUREIkoKIiISUVIQEZGIkoLkFT0/QSRZSgoiIhJRUhARkYiSgoiIRJQUREQkoqQgIiIRJQUREYloltRSl6fPUtBtqSLJSDspmNkZwPnh4mPu/nQ8IYnkVm1ZDXP2LQD0aE6RtJqPzOxq4FfAMeHPL83sm3EGJpKk2rKavK1FicQp3ZrCV4GJ7v4OgJn9AHgc+FFcgYmISO6lmxQMOJCyfCBcJ4VK34LbaevDaGtGEilV6SaF/ws8aWYPhMuXAD+PJyTJtfrG5qRDEJE8kVZScPdbzawB+Hi46svu/ofYohIRkUQcNimY2VHu/raZDQO2hT9t7w1z9zfjDU9ERHKpu5pCHfBpYAPgKestXP5ITHGJiEgCDpsU3P3T4b+jchOOiIgkKd1xCg+ns05ERApbd30K5cBAYISZHc0Ht6EeBRwfc2wiOafpNaTUdden8DXgGuA4gn6FtqTwNvDjGOMSEZEEdNensBhYbGbfdHeNXhYRKXLpjlP4kZmdCowBylPW/yKuwEREJPfSSgpm9s9ANUFSWA1cBPwXoKQgRau+sZk54aypmjFVSkW6D9m5FPgk8Jq7fxk4AxgSW1QiIpKIdJNCq7sfBPab2VHATuCE+MISEZEkdNt8ZGYGPGNmQ4GfEdyF1EIwdbaIiBSRbpOCu7uZTXD3t4C7zOw/gaPc/Zn4w5O4aGbU9AUP3PkFzLo36VBEYpdu89FGMxsP4O7blBCkVGgwm5SadJ+nMBH4opn9CXiHcEI8dz89tshERCTn0k0KU3tycDObBiwG+gK17r6ow/vfAuYA+4HXga+4+596ci4REem9dAevZXyhNrO+wB3AZKAJWGdmK939+ZTN/gBUufu7ZvZ14BZAz4kUEUlIun0KPTEBeMndt7r7+8ByYEbqBu7+iLu/Gy4+AYyMMR4REelGnEnheOCVlOUmDj+z6leB/4gxHpHeqZsZ/IgUMXP37rfqyYHNLgWmufuccPlyYKK7z+9k2y8B84G/dvf3Onl/HjAPoKKiYtzy5cszjqelpYVBgwZlvF+hi8r95tZgxbDgYXl7dryYYFTxO1g+jD6t2Xta7ODylJbWYfn7wMGS/5yXmEzKPWnSpA3uXtXddul2NPfEdtqPeh4ZrmvHzC4EvkMXCQHA3ZcCSwGqqqq8uro642AaGhroyX6FLip33U+CFdXBvfb119+YWEy5sHf0LAY01mXteNWjK1IW8ne8Qsl/zktMHOWOMymsA04ys1EEyeAyYFbqBmZ2JvBTghrFzhhjEemV1MF+FyYYh0jcYutTcPf9BE1Ca4BG4D5332xmN5vZ9HCzGmAQ8Gsz22RmK+OKR0REuhdnTQF3X00w1XbquhtSXutLl4hIHonz7iMRESkwSgoiIhKJtflI8kvqk8RqyxIORkTykmoKIj2lgWxShJQUREQkoqQgIiIRJQUREYmoo1kkQ5VRZ32zRjdL0VFNQUREIkoKIiISUVIQEZGIkoKIiESUFEREJKK7j4rcs9t3M3vhKmrLgucB1JbVJByRiOQz1RRERCSipCDSG3UzNQeSFBUlBRERiSgpiIhIRElBJJvUnCQFTkmhBOiOo4QoQUgBUlIQyZLKhauob2ymvrE5mjRPpNBonIJINtTNjMaCiBQyJQWRDKk5ToqZmo9EYqKmJClESgoiIhJR85FIL9Q3qh9BiotqCiIiElFSEBGRiJqPilBqx+a1pyUYiIgUHNUUilhtWQ2VpjZvEUmfkoKIiESUFEREJKKkICIiESUFERGJKCmIiEhESUFERCIap1BMoge6XJFoGHKoyoWrotlVL/xeQ7LBiByGagoiMdD02lKoYk0KZjbNzP5oZi+Z2cJO3r/AzDaa2X4zuzTOWETyih7VKXkqtuYjM+sL3AFMBpqAdWa20t2fT9nsz8Bs4B/iikMkaao1SCGJs09hAvCSu28FMLPlwAwgSgruvi1872CMcYiISJrM3eM5cNAcNM3d54TLlwMT3X1+J9suA/7d3Vd0cax5wDyAioqKccuXL884npaWFgYNGpTxfgXlza0APLt3OACV1szB8mH0aX0zyagSkc/lHnzcydHfimEfyeqxS+Jz3gmVu3uTJk3a4O5V3W1XEHcfuftSYClAVVWVV1dXZ3yMhoYGerJfQan7CQD7X/hgEry9o2cxoLEuqYgSk8/l3t/4wets34lUEp/zTqjc2RNnUtgOnJCyPDJcJzEIbnnUjKgi0jtx3n20DjjJzEaZWX/gMmBljOcTEZFeii0puPt+YD6wBmgE7nP3zWZ2s5lNBzCz8WbWBHwe+KmZbY4rHhER6V6sfQruvhpY3WHdDSmv1xE0K4mISB7QiGYREYkoKYiISERJQUREIkoKIiISKYjBa9K5yoWrkg5BRIqMkkKR0KRrhaktsdeW1XDh6AqYdW/CEUmpU/ORiIhEVFMocKohiEg2qaYgIiIRJQUREYkoKYjkEz2iUxKmpCCSJ+obm6lvbNatxpIodTQXGF0wRCROqimIiEhESUFERCJKCiJ5prasRh3Okhj1KYgkSIMPJd8oKRQoXUyKW31jM3PCmwq2Lbo44WiklKj5SEREIqopFBjVEErHB39r1RQkd1RTEBGRiGoKBSB1wFptWYKBiEjRU01BREQiqikUCPUliEguKCmI5LnU5kPdnipxU/ORiIhEVFPIU+07l9V0VMra//1VU5B4KSnkMSUDEck1NR+JiEhESSGf1M3U7Jgikig1H4kUEN2JJHFTTUGkgNSW1aivSWKlmkIeqW9sBoimTBYRyTXVFEQKkJ7OJnFRTSFfpPwHV/OApOOQB/HUzYTyqUB1onFJYVNSSEqYBCqfuQKA2rLmJKMREQGUFBJRuXCVkoCI5KVY+xTMbJqZ/dHMXjKzhZ28f4SZ3Ru+/6SZVcYZj0ixqly4ivrGZva07k86FClwsdUUzKwvcAcwGWgC1pnZSnd/PmWzrwJ/cfePmtllwA+Aouk9q+zkLqLglsL2yyK9lfo5qr++GoA5+xa020bjGiQdcTYfTQBecvetAGa2HJgBpCaFGcCN4esVwI/NzNzdY4wrZzpe8Dv+JxXprcN9qWh7r+1zFzRbHvqZjJJF280Os+7NfqBSMOJMCscDr6QsNwETu9rG3feb2W5gOPBGjHH1WGff/A9Hj86UfNBd4qhc2PY67OfqpKYRVy2jqxHaGrmdHIvrS7mZXQpMc/c54fLlwER3n5+yzXPhNk3h8svhNm90ONY8YF64+DHgjz0IaQR5mmxipnKXFpW7tGRS7r9y9w91t1GcNYXtwAkpyyPDdZ1t02Rm/YAhwK6OB3L3pcDS3gRjZuvdvao3xyhEKndpUblLSxzljvPuo3XASWY2ysz6A5cBKztssxK4Mnx9KfDbYulPEBEpRLHVFMI+gvnAGqAvcLe7bzazm4H17r4S+Dnw/8zsJeBNgsQhIiIJiXXwmruvBlZ3WHdDyutW4PNxxpCiV81PBUzlLi0qd2nJerlj62gWEZHCo1lSRUQkUpRJwczuNrOd4S2vbeuGmdlDZrYl/PfoJGOMQxflrjGzF8zsGTN7wMyGJhljHDord8p715qZm9mIJGKLU1flNrNvhn/zzWZ2S1LxxaWLz/lYM3vCzDaZ2Xozm5BkjHEwsxPM7BEzez78214drs/qta0okwKwDJjWYd1C4GF3Pwl4OFwuNss4tNwPAae6++nAi8B1uQ4qB5ZxaLkxsxOAKcCfcx1QjiyjQ7nNbBLBTAFnuPspwP9JIK64LePQv/ctwE3uPha4IVwuNvuBa919DHA2cJWZjSHL17aiTAru/ijB3UypZgD/Gr7+V+CSnAaVA52V293XunvbLGlPEIwXKSpd/L0BbgP+ESjKjrMuyv11YJG7vxduszPngcWsi3I7cFT4egiwI6dB5YC7v+ruG8PXe4BGglkhsnptK8qk0IUKd381fP0aUJFkMAn5CvAfSQeRC2Y2A9ju7k8nHUuOnQycH846/DszG590QDlyDVBjZq8Q1I6KsUYcCWeUPhN4kixf20opKUTCAXJF+e2xK2b2HYLq56+SjiVuZjYQ+CeCZoRS0w8YRtC8sAC4z8ws2ZBy4uvA37v7CcDfE4yBKkpmNgi4H7jG3d9OfS8b17ZSSgrNZnYsQPhv0VWru2Jms4FPA18skRHjJwKjgKfNbBtBk9lGM/twolHlRhPwGw88BRwkmB+n2F0J/CZ8/WuCWZqLjpmVESSEX7l7W3mzem0rpaSQOqXGlcC/JRhLzpjZNIJ29enu/m7S8eSCuz/r7se4e6W7VxJcKM9y99cSDi0XHgQmAZjZyUB/SmOiuB3AX4evPwFsSTCWWIQ1vp8Dje5+a8pb2b22uXvR/QD3AK8C+wguCF8lmJL7YYIPSz0wLOk4c1TulwimJ98U/tyVdJy5KHeH97cBI5KOM0d/7/7AL4HngI3AJ5KOM0fl/jiwAXiaoJ19XNJxxlDujxM0DT2T8v/5U9m+tmlEs4iIREqp+UhERLqhpCAiIhElBRERiSgpiIhIRElBREQiSgqSd8yspZf7rzCzj2S4z9+a2RW9OOddZnZeT/ePk5ldYGYbzWy/mV2asv5DZvafScYm+UdJQYqKmZ0C9HX3rRns08/d73L3X/Ti1GcTTDiY1vl6cZ6Ox6o2s2XdbPZnYDZQl7rS3V8HXs3XZCbJUFKQvGWBGjN7zsyeNbOZ4fo+ZnZn+MyAh8xsdco34C+SMqLTzFrM7LZw/vmHzexD4foGM7vdzNYDV5vZjWb2D+F7HzWzejN7OvyGfWK4foGZrQufTXFTyjlGAy+6+wEzmxtu87SZ3R/Ow4SZLQtrE08Ct5jZkeFzAZ4ysz+EE/hhZpVm9lh43o1mdm5vf4/uvs3dnyGY8qKjB8PfmQigpCD57bPAWOAM4EKCWTCPDddXAmOAy4FzUvY5j2Bka5sjgfUePFvgd8A/p7zX392r3P2HHc77K+AOdz8DOJfg2/QU4CSCOXXGAuPM7IJw+4uAtmaY37j7+HDfRoLRtm1GAue6+7eA7wC/dfcJBNNS1JjZkQTz1kx297OAmcCS9H5VPbYeOD/mc0gByVo1ViQGHwfucfcDBJN+/Q4YH67/tbsfBF4zs0dS9jkWeD1l+SBwb/j6l3wwaRop6yNmNhg43t0fAHD31nD9FIIH9vwh3HQQQZJ4FJgKfDlcf6qZ/W9gaLjNmpTD/zosC+GxprfVToBy4H8QzOHzYzMbCxwgmAr7EGGN44jwHMPMbFP41rfdfU1n+3RhJ3BcBttLkVNSkGKzl+AC25XUeV3eyeC4BvyLu/+03cqgeWiou7c91GUZcIm7Px3OTlvdxfkM+Jy7/7HD8W4EmglqR32A1k4L4T4x3L4amO3uszMoS6pygt+ZCKDmI8lvjwEzzaxv2BdwAfAU8N/A58K+hQraX3gbgY+mLPcB2vobZgH/dbgTevBEqyYzuwTAzI4IL/xrgK+Ec9ljZseb2TEETT+pNZXBBM1NZRy+rX4N8M22Zx2Y2Znh+iHAq2Et6HKg7+HizYKTCSbPEwGUFCS/PUAwI+TTwG+Bf/Rg+uv7CWbHfJ6gSWgjsDvcZxWHfjufYMFD3j8B3JzGeS8H/s7MngF+D3zY3dcS3L3zuJk9C6wgSACp/QkA1xPM0vnfwAuHOcf3gDLgGTPbHC4D3AlcaWZPA/+TzGoznTKz8WbWBHwe+Gl4vjaTCH5nIgCaJVUKk5kNcvcWMxtOUHs4z91fM7MBBN/czwvvBmpx90ExxrERmOju++I6R5zM7FFghrv/JelYJD8oKUhBMrMGgs7c/sAt7r4s5b2pBA8i+XPcSaGQhU1y57n7g0nHIvlDSUFERCLqUxARkYiSgoiIRJQUREQkoqQgIiIRJQUREYkoKYiISOT/A6uLduGyB59kAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHHNJREFUeJzt3X+YVOV99/H3V1xcEQSBuI1iu2g0z+IvlBWTGO3Q+GONucA8NUJRE9sASSM+2ppNyFPRaP8oyTZWaI2Gbii1ZosGo+UqtOJaNton/uBHMAqrQrg2cTHBiIWwBJSV7/PHnD2cXXZ2ZnfnzJmZ/byui8s559xzzvfeXec7932fc9/m7oiIiAAck3QAIiJSPJQUREQkpKQgIiIhJQUREQkpKYiISEhJQUREQkoKIiISUlIQEZGQkoKIiISOTTqA/ho/frxXV1dnPL5//35OOOGEwgWUMNW3/A21Oqu+8di4ceM77v6hbOVKLilUV1ezYcOGjMdbWlpIpVKFCyhhqm/5G2p1Vn3jYWa/yKWcuo9ERCSkpCAiIiElBRERCZXcmIKIlKdDhw7R3t7O6NGjaW1tTTqcgsl3fSsrK5kwYQIVFRUDer+SgogUhfb2dkaNGsW4ceM48cQTkw6nYPbt28eoUaPyci53Z/fu3bS3tzNx4sQBnUPdRyJSFA4ePMi4ceMws6RDKVlmxrhx4zh48OCAz6GkICJFQwlh8Ab7M1RSEBGRkMYURKQoVS9YndfztS26ps/je/bsoampia985Sv9PvenP/1pmpqaGDNmTE7ln3zySc466ywmTZrU72vFTS0FSUT1gtXhP5FisGfPHr773e/2eqyzs7PP965ZsybnhADppLB169YBXStuSgoiIsCCBQv4+c9/zuTJk6mvr6elpYVLL72U6dOnh9/or732WqZMmcLZZ5/N0qVLw/dWV1fzzjvv0NbWRk1NDXPnzuXss8/myiuv5MCBA92u85Of/IRVq1ZRX1/P5MmT2bFjB6lUittvv53a2loWL17MzTffzMqVK8P3jBw5Mnzd0NDARRddxHnnncfdd9+d95+DkoKICLBo0SLOOOMMNm/eTENDAwCbNm1i8eLFvPHGGwAsW7aMjRs3smHDBpYsWcLu3buPOs+2bdu45ZZb2LJlC2PGjOHxxx/vdvwTn/gE06dPp6Ghgc2bN3P66acD8P7777NhwwbuuOOOjDGuXbuWbdu28dJLL7F582Y2btzIs88+m68fAaAxBRGRjKZOndrtfv8lS5bwxBNPAPDmm2+ybds2xo0b1+09EydOZPLkyQBMmTKFtra2nK41c+bMrGXWrl3L2rVrueCCCwDo6Ohg27ZtXHbZZTldIxdKCiIiGUSntG5paaG5uZnnn3+eESNGkEqlen0e4LjjjgtfDxs27Kjuo1yudeyxx3L48GEADh8+zPvvvw+kH077xje+wZe+9KUB1ScX6j4SEQFGjRrFvn37Mh7fu3cvJ510EiNGjOC1117jhRdeiO1a1dXVbNy4EYBVq1Zx6NAhAK666iqWLVtGR0cHADt37uTtt98ecBy9UUtBRIpStltI823cuHFccsklnHPOOVx99dVcc03369fV1fHQQw9RU1PDRz/6UT72sY8N+FqzZs1i7ty5LFmyhOXLlx91fO7cucyYMYPzzz+furq6sBVx5ZVX0traysc//nEgPQD9yCOPcPLJJw84lp7M3fN2skKora11LbJzRKnWN3oran/+5y/V+g7GUKlza2srNTU1eZ0LqBTEUd+un2WUmW1099ps71X3kYiIhJQUREQkpKQgIiIhJQUREQkpKYiISEhJQUREQnpOQUSKU1P2aR/6ZfajfR4ezNTZ999/P/PmzWPEiBEDja5oqKUgIkLfU2dnc//99/O73/0uzxElQy0FERG6T519xRVXcPLJJ/PYY4/x3nvv8dnPfpZ77rmH/fv3c/3119Pe3s4HH3zAwoUL2bVrF2+99RbTpk1j/PjxrFu3LumqDIqSgogI6amzX331VTZv3szatWtZuXIlL730Eu7O9OnTefbZZ/nNb37DKaecwurV6Sfy9+7dy+jRo7nvvvtYt24d48ePT7gWg6fuIxGRHqJTVF944YW89tprbNu2jXPPPZenn36ar3/96zz33HOMHj066VDzTi0FEZEe+pqietOmTaxZs4Y777yTT33qU9x1110JRBgftRREROg+nXWmKarfeustRowYwY033kh9fT2bNm066r2lTi0FESlOWW4hzbeeU2fPnj37qCmqt2/fTn19PccccwwVFRU8+OCDAMybN4+6ujpOOeUUDTSLiJSLpqambtu33XZbt+0zzjiDq6666qj33Xrrrdx6662xxlYo6j4SEZGQkoKIiIRiTQpmVmdmr5vZdjNb0Ee5PzYzN7OsqwKJSPkqtZUgi9Fgf4axjSmY2TDgAeAKoB1Yb2ar3H1rj3KjgNuAF+OKRYpM00waK3YBMOdQfcLBSLGorKxk9+7dDB8+POlQSpa7s3v3biorKwd8jjgHmqcC2919B4CZrQBmAFt7lPtr4FuAPh1EhrAJEybQ3t7Onj17BvWhVmoOHjyY1/pWVlYyYcKEAb8/zqRwKvBmZLsduDhawMwuBE5z99VmpqQgMoRVVFQwceJEWlpauOCCC5IOp2CKrb6J3ZJqZscA9wE351B2HjAPoKqqipaWloxlOzo6+jxebkqyvpVXcaCmE4A7vLNf8ZdkfQdpqNVZ9U1WnElhJ3BaZHtCsK/LKOAcoMXMAH4PWGVm0919Q/RE7r4UWApQW1vrqVQq40VbWlro63i5Kcn6Nj1Ic2t6TOE7h+ppuyGV81tLsr6DNNTqrPomK86ksB4408wmkk4Gs4DZXQfdfS8QTiloZi3AV3smBBkioguqFPhJVhE5IrZbUt29E5gPPAW0Ao+5+xYzu9fMpsd1XRERGbhYxxTcfQ2wpse+XqcUdPdUnLGIiEh2mvtICiPf6+2KSCw0zYWIiISUFEREJKSkICIiISUFEREJaaBZ8qJ6werwdduiaxKMREQGQ0lBYtWVLBordnF5TVXC0YhINuo+EhGRkJKCiIiE1H0kiatesDpcdAdgTtDlpLEJkcJTUpC8iw46i0hpUVKQ2DRWNCQdgoj0k5KCJEqJQ6S4aKBZRERCSgoiIhJS95HklbqDREqbkoLkhZKBSHlQ95GIiISUFEREJKSkICIiIY0pSL9oimyR8qaWgoiIhNRSkOyaZkY2Pp9YGCISPyUFyaq5dVf2QjHr6ra649xOUsmGIlLW1H0kIiIhJQUREQmp+0iKTvh0dNPDMPvRZIMRGWLUUhARkZBaClIwxTBgLSJ9U0tBRERCainIwHV7fkFEyoFaCiIiElJSEBGRkJKCiIiElBRERCSkpCAiIqFY7z4yszpgMTAMaHT3RT2Ofxm4BfgA6ADmufvWOGOSzHJZK6H7WsxVMUckIoUWW0vBzIYBDwBXA5OAPzGzST2KNbn7ue4+Gfg2cF9c8YiISHZxthSmAtvdfQeAma0AZgBhS8DdfxspfwLgMcYjOTjSEii+VdUaKxo4YLOTDkOkrJl7PJ/DZnYdUOfuc4Ltm4CL3X1+j3K3AH8JDAf+yN239XKuecA8gKqqqikrVqzIeN2Ojg5GjhyZt3oUu3zW95Wde6m29FQUo045K9y/7603ei0/qvLId4p9BzvzEsNR5x97erfYDleOZfTY8Xm/VjHT33R5K1R9p02bttHda7OVS/yJZnd/AHjAzGYDdwJf6KXMUmApQG1tradSqYzna2lpoa/j5Saf9b15wWoaK5oASM1uCfc3L/xmr+WjaeD4vETQXaqmClKPdovtQM1sUqnrwjJDYc1o/U2Xt2Krb5xJYSdwWmR7QrAvkxXAgzHGI6UomEqjseLIZHrRRCAi+RXnLanrgTPNbKKZDQdmAauiBczszMjmNcBRXUciIlI4sbUU3L3TzOYDT5G+JXWZu28xs3uBDe6+CphvZpcDh4D/oZeuIxERKZxYxxTcfQ2wpse+uyKvb4vz+iIi0j+JDzSLDEb3h+nKc6BZpJCUFKR3WitBZEhSUpCS0711ICL5pAnxREQkpKQgIiIhJQUREQkpKYiISEhJQUREQkoKIiISUlIQEZFQzs8pmNn5wKXB5nPu/nI8IYmISFJySgpmdhswF/hRsOsRM1vq7n8fW2Qy5DW37speSETyKteWwhdJr5q2H8DMvgU8DygpiIiUkVzHFAz4ILL9QbBPRETKSK4thX8CXjSzJ4Lta4HvxxOSiIgkJaek4O73mVkL8Mlg15+6+09ji0oKq5clL0VkaOozKZjZie7+WzMbC7QF/7qOjXX3d+MNT0RECilbS6EJ+AywEfDIfgu2T48pLhERSUCfScHdPxP8d2JhwhEZhOjCQLMfTS4OkRKW091HZvZMLvtERKS0ZRtTqARGAOPN7CSO3IZ6InBqzLFJvumbtIhkkW1M4UvA7cAppMcVupLCb4F/iDEuKZDqBat115GIhLKNKSwGFpvZrZrSQkSk/OX6nMLfm9k5wCSgMrL/4bgCExGRwst1Qry7gRTppLAGuBr4b0BJQYpGdAK9yxOMQ6SU5TrNxXXA+cBP3f1PzawKeCS+sCROza27mLNgddJhiEgRynVCvIPufhjoNLMTgbeB0+ILS0REkpC1pWBmBvzMzMYA/0j6LqQO0lNnixSl6khLqG3RNQlGIlJasiYFd3czm+rue4CHzOw/gRPd/WfxhyciIoWU65jCJjO7yN3Xu3tbnAFJDKIPrYmI9CHXpHAxcIOZ/QLYTzAhnrufF1tkIvnSlRT1FLdIVrkmhatijUIKrrGiIekQRKQI5frw2i/iDkT6p2sg9Y5zO0n1VmCIdxl1T3pVicUhUmpyvSVVRESGgFy7j6REdLUgGit2cXmNviH3VN3joT3drirSXawtBTOrM7PXzWy7mS3o5fhfmtlWM/uZmT1jZn8QZzwiItK32JKCmQ0DHiA9T9Ik4E/MbFKPYj8FaoO7mFYC344rHhERyS7O7qOpwHZ33wFgZiuAGcDWrgLuvi5S/gXgxhjjKX9NMzOujRCdLE5EJJM4k8KpwJuR7XbSzztk8kXgP2KMR4a6SNKcc6g+4WBEipO5ezwnNrsOqHP3OcH2TcDF7j6/l7I3AvOBP3T393o5Pg+YB1BVVTVlxYoVGa/b0dHByJEj81OJIvbKzr0AVB0PJ48dnd757g72HezMy/lHVR75vpCvc+bD4cqxHHPw3UGfp83Tg/Dnnjp60OeK21D5m+6i+sZj2rRpG929Nlu5OFsKO+k+k+qEYF83ZnY58FdkSAgA7r4UWApQW1vrqVQq40VbWlro63i5uDnynML1XfVtejBv3USpyJ1LxdT1dKBmNse3Ng36PN8JWgptN6QGfa64DZW/6S6qb7LiTArrgTPNbCLpZDALmB0tYGYXAN8j3aJ4O8ZYpJ+KKRGISOHElhTcvdPM5gNPAcOAZe6+xczuBTa4+yqgARgJ/DA9Qze/dPfpccVUjqpt15B/ellE8ifWh9fcfQ3p5Tuj++6KvNaqiSIiRUTTXIiISEhJQUREQkoKIiISUlIQEZGQkoKIiISUFEREJKSkICIiISUFEREJKSmIiEhISUEkk6aZmkJEhhyt0VyKIusCHOg+x6CIyKCopSAiIiG1FGRIaqxoSL9oehhmP5psMCJFRC0FEREJqaUgEqgOVrMDaFt0TYKRiCRHSUEkuMOosWIXc4JlOkWGKiUFGdK07KhId0oKIr2oXrA6vO1XywPKUKKBZhERCamlUEr0dK2IxEwtBRERCSkpiIhISN1HxU5dRgUVPumcgZ5lkHKnloKIiISUFEREJKSkICIiISUFEREJaaC5SGgAU0SKgVoKIiISUkuhyEUnbLu8piprGRGRwVBSEMmm27Min08sDJFCUFIoIWoRiEjclBRE+qH7E8+6IUDKj5KCyEBFu5VmP5pcHCJ5pLuPREQkpJaCSB7oORMpF7G2FMyszsxeN7PtZragl+OXmdkmM+s0s+vijEVERLKLraVgZsOAB4ArgHZgvZmtcvetkWK/BG4GvhpXHCVJ02WXBN0NJuUozu6jqcB2d98BYGYrgBlAmBTcvS04djjGOEQGRR/+MpSYu8dz4nR3UJ27zwm2bwIudvf5vZRdDvy7u6/McK55wDyAqqqqKStWrMh43Y6ODkaOHDn4ChTYKzv3hq/PPX53+Hrfwc4+33e4cizHHHw3triKTbHWt82PPG1+7qmj83ruUv2bHijVNx7Tpk3b6O612cqVxECzuy8FlgLU1tZ6KpXKWLalpYW+jhermyMDlY0VD4evj8/yvgM1szm+tSmmqIpPsdb3O4fqw9dtN6Tyeu5S/ZseKNU3WXEONO8ETotsTwj2iYhIkYozKawHzjSziWY2HJgFrIrxeiIiMkixJQV37wTmA08BrcBj7r7FzO41s+kAZnaRmbUDnwO+Z2Zb4opHRESyi3VMwd3XAGt67Lsr8no96W4lkZLWNSfSnMjYgkgpKomBZpFyoKeepRRo7iORPGqsaKB5YYrmhaluSUCkVKilUCS6T8ksIpIMJYUCUxfC0KLWgpQaJYWkaZ4jESkiGlMQEZGQWgoiMYmOE0VvVW2saICmYCoTrdgmRUYtBRERCamlIFIAurtMSoWSQqEEA8qNFbv01KuIFC11H4mISEhJQUREQuo+KoDqBatprNCSjiJS/JQUEhAOOjY93HdBEZECU/eRSAKaW3fR3LpL02BI0VFSEBGRkLqPRBKmSRKlmKilICIiISUFEREJqftIJEE9p79oXtgQPvGuriRJgpJCgppb9eyCiBQXdR+JiEhILYU80l0kkndNM6HyKmh6UGsvSEEoKcQpMjOqSH91TY9yoKYz6VBkCFFSECkyva290Ny6izlBS1StUImTxhRERCSklkJMNDOqiJQiJYUB0ICyJKHb7LoadJaYKCnkmdbilYIIbmKIJgd9WZF8UFLIkaY4lmLUvDAVvm6siB5RUpCBUVLop2hLoGs6ApFC0pPwEiclBZEypK4kGSglhUFSt5KIlBMlhUHQoLKUgqythq5Ba9BdTaKkkJOmmXrmQEpKb19Yeo6BdSWLxopdXF5TdfRJernDScpfrEnBzOqAxcAwoNHdF/U4fhzwMDAF2A3MdPe2OGPKKvKtSQN6Uk4aKxpoXtgQ2T5yLPxbj9zNBPSeLLJRMilpsSUFMxsGPABcAbQD681slbtvjRT7IvA/7v4RM5sFfAuYefTZCkeJQOSITHMuZRpL06B26YuzpTAV2O7uOwDMbAUwA4gmhRnAN4PXK4F/MDNzd48xrqNE/8C73+stIl1dUZlaGVHRMnNyuAmj69xzDtUXZrxDrZis4kwKpwJvRrbbgYszlXH3TjPbC4wD3oklogxdQ0oEIvnXnxsxol1bB2pm07zwm+GxaBdWzxZKb9c4qssrSADd5iOLdJNl7CLLljjKdIDe4vpSbmbXAXXuPifYvgm42N3nR8q8GpRpD7Z/HpR5p8e55gHzgs2PAq/3cenxxJVUipPqW/6GWp1V33j8gbt/KFuhOFsKO4HTItsTgn29lWk3s2OB0aQHnLtx96XA0lwuamYb3L12QBGXINW3/A21Oqu+yYpzPYX1wJlmNtHMhgOzgFU9yqwCvhC8vg74r0KPJ4iIyBGxtRSCMYL5wFOkb0ld5u5bzOxeYIO7rwK+D/yLmW0H3iWdOEREJCGxPqfg7muANT323RV5fRD4XJ4vm1M3UxlRfcvfUKuz6pug2AaaRUSk9GiNZhERCZVsUjCzZWb2dnBba9e+BjN7zcx+ZmZPmNmYJGPMt97qHDl2h5m5mY1PIrY4ZKqvmd0a/J63mNm3k4ov3zL8TU82sxfMbLOZbTCzqUnGmE9mdpqZrTOzrcHv8rZg/1gze9rMtgX/PSnpWPOhj/oW1edWySYFYDlQ12Pf08A57n4e8AbwjUIHFbPlHF1nzOw04Ergl4UOKGbL6VFfM5tG+kn48939bOBvE4grLss5+vf7beAed58M3BVsl4tO4A53nwR8DLjFzCYBC4Bn3P1M4Jlguxxkqm9RfW6VbFJw92dJ37EU3bfW3TuDzRdIPxtRNnqrc+DvgK8BZTVAlKG+fw4scvf3gjJvFzywmGSorwMnBq9HA28VNKgYufuv3H1T8Hof0Ep6loMZwD8Hxf4ZuDaZCPMrU32L7XOrZJNCDv4M+I+kg4ibmc0Adrr7y0nHUiBnAZea2Ytm9mMzuyjpgGJ2O9BgZm+SbhWVW+sXADOrBi4AXgSq3P1XwaFfAwOYqrW49ahvVOKfW2WZFMzsr0g31X6QdCxxMrMRwP8l3a0wVBwLjCXd/K4HHjMzSzakWP058BfufhrwF6Sf7SkrZjYSeBy43d1/Gz0WPMxaVi3gTPUtls+tsksKZnYz8BnghiHwdPQZwETgZTNrI93s3GRmv5doVPFqB37kaS8Bh0nPHVOuvgD8KHj9Q9KzD5cNM6sg/QH5A3fvqucuM/twcPzDQNl0EWaob1F9bpVVUggW9fkaMN3df5d0PHFz91fc/WR3r3b3atIfmBe6+68TDi1OTwLTAMzsLGA45T152lvAHwav/wjYlmAseRW08L4PtLr7fZFD0elvvgD8W6Fji0Om+hbb51bJPrxmZv8KpEh/S9wF3E26v/U4jkyq94K7fzmRAGPQW53d/fuR421Abc9ZZktVht/xvwDLgMnA+8BX3f2/kooxnzLU93XSqxceCxwEvuLuG5OKMZ/M7JPAc8ArpFt8kO4OfRF4DPh94BfA9e7e2w0WJaWP+i6hiD63SjYpiIhI/pVV95GIiAyOkoKIiISUFEREJKSkICIiISUFEREJKSlIWTCzjkG+f6WZnd7P93zZzD4/mOv2l5n9LzN73szeM7OvRvYPN7Nng7XORQZMSUGGPDM7Gxjm7jv68Z5j3f0hd384j3FUm1lLlmLvAv+HHrPDuvv7pGcUnZmveGRoUlKQsmJpDWb2qpm9YmYzg/3HmNl3g3nrnzazNWZ2XfC2G4g8NWtmHWb2d8Gc98+Y2YeC/S1mdr+ZbQBuM7Nvdn1bN7OPmFmzmb1sZpvM7Ixgf72ZrQ/myr9nsPVz97fdfT1wqJfDTwZ1ERkwJQUpN/+b9NPO5wOXk55h9MPB/mpgEnAT8PHIey4Bok8JnwBsCNZr+DHpJ4u7DHf3Wnf/To/r/gB4wN3PBz4B/MrMrgTOJD1f0WRgipldlpda9u5VoNxnjZWYqf9Rys0ngX919w9IT6z2Y9IflJ8Efujuh4Ffm9m6yHs+DPwmsn0YeDR4/QhHJqQjsj9kZqNIz4v/BIC7Hwz2X0l68aOfBkVHkk4Sz/Z4/xOkJzYcDvy+mW0ODi1293/KteLu/oGZvW9mo4L5+kX6TUlBBA4AlX0cj84Fs78f5zXgb9z9e30VcvfPQjjH/nJ3T/XjGj0dR3qOJJEBUfeRlJvngJlmNiwYC7gMeAn4f8AfB2MLVaQnnuvSCnwksn0M0DXeMBv4774uGHwrbzezawHM7LhgrYungD8L5s/HzE41s5MHW8FMzGwc8I679zbeIJITtRSk3DxBerzgZdLf8L/m7r82s8eBTwFbgTeBTcDe4D2rSSeJ5mB7PzDVzO4kPZd/Lnf03AR8z8zuJT0I/Dl3X2tmNcDzwTpAHcCNDGJ9gGCtjA2kl+g8bGa3A5OCxVqmBXURGTDNkipDhpmNdPeO4Bv1S8AlQcI4HlgXbH9gZh3uPjLZaPvPzH4ELHD3N5KORUqXWgoylPy7mY0hPaD7112LEbn7ATO7m/Si8b9MMsCBMrPhwJNKCDJYaimIiEhIA80iIhJSUhARkZCSgoiIhJQUREQkpKQgIiIhJQUREQn9fyWsTJQulZ53AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(cv['log_parea_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price'] / test['building_area']), bins=100, label='test',\n",
    "         normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price/area + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()\n",
    "\n",
    "plt.hist(cv['log_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price']), bins=100, label='test', normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.388185e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.924688e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.115398e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>6.035152e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.064497e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.236378e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.749268e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.095208e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.974268e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.117081e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.388185e+07\n",
       "1     BTshNOJyKHnT2YIT  3.924688e+06\n",
       "2     dhdymr0lV8N5kZOT  1.115398e+07\n",
       "3     VEwyGGMcD56w5BOc  6.035152e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.064497e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.236378e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.749268e+07\n",
       "9997  YunNwAhcqkf6YclI  1.095208e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.974268e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.117081e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "test[['building_id', 'total_price']].check()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output file to: output/stack_mean_11_1-11,14-17.csv\n"
     ]
    }
   ],
   "source": [
    "output_filename = 'output/stack_mean_{}_{}.csv'.format(stack_idx, models)\n",
    "print('Output file to:', output_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[['building_id', 'total_price']].to_csv(output_filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.388185e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.924688e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.115398e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>6.035152e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.064497e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.236378e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.749268e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.095208e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.974268e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.117081e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.388185e+07\n",
       "1     BTshNOJyKHnT2YIT  3.924688e+06\n",
       "2     dhdymr0lV8N5kZOT  1.115398e+07\n",
       "3     VEwyGGMcD56w5BOc  6.035152e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.064497e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.236378e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.749268e+07\n",
       "9997  YunNwAhcqkf6YclI  1.095208e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.974268e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.117081e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "pd.read_csv(output_filename).check()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}