notebooks/2_calendar_only.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Category 2: Using the calendar features only\n",
    "The second model category will only use calendar features (dummies for holiday, weekday, hour and month) to create a forecast for the electricity load."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model category specific configuration\n",
    "These parameters are model category specific\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Model category name used throughout the subsequent analysis\n",
    "model_cat_id = \"02\"\n",
    "\n",
    "# Which features from the dataset should be loaded:\n",
    "# ['all', 'actual', 'entsoe', 'weather_t', 'weather_i', 'holiday', 'weekday', 'hour', 'month']\n",
    "features = ['actual', 'calendar']\n",
    "\n",
    "# LSTM Layer configuration\n",
    "# ========================\n",
    "# Stateful True or false\n",
    "layer_conf = [ True, True, True ]\n",
    "# Number of neurons per layer\n",
    "cells = [[ 5, 10, 20, 30, 50, 75, 100, 125, 150 ], [0, 10, 20, 50], [0, 10, 15, 20]]\n",
    "# Regularization per layer\n",
    "dropout = [0, 0.1, 0.2]\n",
    "# Size of how many samples are used for one forward/backward pass\n",
    "batch_size = [8]\n",
    "# In a sense this is the output neuron dimension, or how many timesteps the neuron should output. Currently not implemented, defaults to 1.\n",
    "timesteps = [1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Module imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import math\n",
    "import itertools\n",
    "import datetime as dt\n",
    "import pytz\n",
    "import time as t\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import read_csv\n",
    "from pandas import datetime\n",
    "from numpy import newaxis\n",
    "\n",
    "import matplotlib as mpl\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as stats\n",
    "from statsmodels.tsa import stattools\n",
    "from tabulate import tabulate\n",
    "\n",
    "import math\n",
    "import keras as keras\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dense, Dropout, LSTM\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.utils import np_utils\n",
    "from keras.models import load_model\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "\n",
    "from IPython.display import HTML\n",
    "from IPython.display import display\n",
    "%matplotlib notebook\n",
    "mpl.rcParams['figure.figsize'] = (9,5)\n",
    "\n",
    "# Import custom module functions\n",
    "module_path = os.path.abspath(os.path.join('../'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "from lstm_load_forecasting import data, lstm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall configuration\n",
    "These parameters are later used, but shouldn't have to change between different model categories (model 1-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Directory with dataset\n",
    "path = os.path.join(os.path.abspath(''), '../data/fulldataset.csv')\n",
    "\n",
    "# Splitdate for train and test data. As the TBATS and ARIMA benchmark needs 2 full cycle of all seasonality, needs to be after jan 01. \n",
    "loc_tz = pytz.timezone('Europe/Zurich')\n",
    "split_date = loc_tz.localize(dt.datetime(2017,2,1,0,0,0,0))\n",
    "\n",
    "# Validation split percentage\n",
    "validation_split = 0.2\n",
    "# How many epochs in total\n",
    "epochs = 30\n",
    "# Set verbosity level. 0 for only per model, 1 for progress bar...\n",
    "verbose = 0\n",
    "\n",
    "# Dataframe containing the relevant data from training of all models\n",
    "results = pd.DataFrame(columns=['model_name', 'config', 'dropout',\n",
    "                                'train_loss', 'train_rmse', 'train_mae', 'train_mape', \n",
    "                                'valid_loss', 'valid_rmse', 'valid_mae', 'valid_mape', \n",
    "                                'test_rmse', 'test_mae', 'test_mape',\n",
    "                                'epochs', 'batch_train', 'input_shape',\n",
    "                                'total_time', 'time_step', 'splits'\n",
    "                               ])\n",
    "# Early stopping parameters\n",
    "early_stopping = True\n",
    "min_delta = 0.006\n",
    "patience = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation and model generation\n",
    "Necessary preliminary steps and then the generation of all possible models based on the settings at the top of this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================\n",
      "| Number of model configs generated | 432 |\n"
     ]
    }
   ],
   "source": [
    "# Generate output folders and files\n",
    "res_dir = '../results/notebook_' + model_cat_id + '/'\n",
    "plot_dir = '../plots/notebook_' + model_cat_id + '/'\n",
    "model_dir = '../models/notebook_' + model_cat_id + '/'\n",
    "os.makedirs(res_dir, exist_ok=True)\n",
    "os.makedirs(model_dir, exist_ok=True)\n",
    "output_table = res_dir + model_cat_id + '_results_' + t.strftime(\"%Y%m%d\") + '.csv'\n",
    "test_output_table = res_dir + model_cat_id + '_test_results' + t.strftime(\"%Y%m%d\") + '.csv'\n",
    "\n",
    "# Generate model combinations\n",
    "models = []\n",
    "models = lstm.generate_combinations(\n",
    "    model_name=model_cat_id + '_', layer_conf=layer_conf, cells=cells, dropout=dropout, \n",
    "    batch_size=batch_size, timesteps=[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the data:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load data and prepare for standardization\n",
    "df = data.load_dataset(path=path, modules=features)\n",
    "df_scaled = df.copy()\n",
    "df_scaled = df_scaled.dropna()\n",
    "\n",
    "# Get all float type columns and standardize them\n",
    "floats = [key for key in dict(df_scaled.dtypes) if dict(df_scaled.dtypes)[key] in ['float64']]\n",
    "scaler = StandardScaler()\n",
    "scaled_columns = scaler.fit_transform(df_scaled[floats])\n",
    "df_scaled[floats] = scaled_columns\n",
    "\n",
    "# Split in train and test dataset\n",
    "df_train = df_scaled.loc[(df_scaled.index < split_date )].copy()\n",
    "df_test = df_scaled.loc[df_scaled.index >= split_date].copy()\n",
    "\n",
    "# Split in features and label data\n",
    "y_train = df_train['actual'].copy()\n",
    "X_train = df_train.drop('actual', 1).copy()\n",
    "y_test = df_test['actual'].copy()\n",
    "X_test = df_test.drop('actual', 1).copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running through all generated models\n",
    "Note: Depending on the above settings, this can take very long!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================= Model 1/288 =========================\n",
      "| Starting with model | 02_1_l-30                  |\n",
      "| Starting time       | 2017-06-17 09:27:58.112810 |\n",
      "========================= Model 2/288 =========================\n",
      "| Starting with model | 02_2_l-30_d-0.1            |\n",
      "| Starting time       | 2017-06-17 09:28:47.857243 |\n",
      "========================= Model 3/288 =========================\n",
      "| Starting with model | 02_3_l-30_d-0.2            |\n",
      "| Starting time       | 2017-06-17 09:31:16.593137 |\n",
      "========================= Model 4/288 =========================\n",
      "| Starting with model | 02_4_l-30_l-10             |\n",
      "| Starting time       | 2017-06-17 09:31:59.621169 |\n",
      "========================= Model 5/288 =========================\n",
      "| Starting with model | 02_5_l-30_l-10_d-0.1       |\n",
      "| Starting time       | 2017-06-17 09:34:11.360011 |\n",
      "========================= Model 6/288 =========================\n",
      "| Starting with model | 02_6_l-30_l-10_d-0.2       |\n",
      "| Starting time       | 2017-06-17 09:37:01.269220 |\n",
      "========================= Model 7/288 =========================\n",
      "| Starting with model | 02_7_l-30_l-15             |\n",
      "| Starting time       | 2017-06-17 09:41:10.714934 |\n",
      "========================= Model 8/288 =========================\n",
      "| Starting with model | 02_8_l-30_l-15_d-0.1       |\n",
      "| Starting time       | 2017-06-17 09:50:01.487827 |\n",
      "========================= Model 9/288 =========================\n",
      "| Starting with model | 02_9_l-30_l-15_d-0.2       |\n",
      "| Starting time       | 2017-06-17 09:54:26.832310 |\n",
      "========================= Model 10/288 =========================\n",
      "| Starting with model | 02_10_l-30_l-20            |\n",
      "| Starting time       | 2017-06-17 09:58:50.437371 |\n",
      "========================= Model 11/288 =========================\n",
      "| Starting with model | 02_11_l-30_l-20_d-0.1      |\n",
      "| Starting time       | 2017-06-17 10:01:12.081681 |\n",
      "========================= Model 12/288 =========================\n",
      "| Starting with model | 02_12_l-30_l-20_d-0.2      |\n",
      "| Starting time       | 2017-06-17 10:04:36.339659 |\n",
      "========================= Model 13/288 =========================\n",
      "| Starting with model | 02_13_l-30_l-10            |\n",
      "| Starting time       | 2017-06-17 10:11:08.030361 |\n",
      "========================= Model 14/288 =========================\n",
      "| Starting with model | 02_14_l-30_l-10_d-0.1      |\n",
      "| Starting time       | 2017-06-17 10:13:04.899830 |\n",
      "========================= Model 15/288 =========================\n",
      "| Starting with model | 02_15_l-30_l-10_d-0.2      |\n",
      "| Starting time       | 2017-06-17 10:17:51.289293 |\n",
      "========================= Model 16/288 =========================\n",
      "| Starting with model | 02_16_l-30_l-10_l-10       |\n",
      "| Starting time       | 2017-06-17 10:21:44.408105 |\n",
      "========================= Model 17/288 =========================\n",
      "| Starting with model | 02_17_l-30_l-10_l-10_d-0.1 |\n",
      "| Starting time       | 2017-06-17 10:24:21.758120 |\n",
      "========================= Model 18/288 =========================\n",
      "| Starting with model | 02_18_l-30_l-10_l-10_d-0.2 |\n",
      "| Starting time       | 2017-06-17 10:30:37.746963 |\n",
      "========================= Model 19/288 =========================\n",
      "| Starting with model | 02_19_l-30_l-10_l-15       |\n",
      "| Starting time       | 2017-06-17 10:38:01.553105 |\n",
      "========================= Model 20/288 =========================\n",
      "| Starting with model | 02_20_l-30_l-10_l-15_d-0.1 |\n",
      "| Starting time       | 2017-06-17 10:41:00.033977 |\n",
      "========================= Model 21/288 =========================\n",
      "| Starting with model | 02_21_l-30_l-10_l-15_d-0.2 |\n",
      "| Starting time       | 2017-06-17 10:51:15.162333 |\n",
      "========================= Model 22/288 =========================\n",
      "| Starting with model | 02_22_l-30_l-10_l-20       |\n",
      "| Starting time       | 2017-06-17 11:02:47.131870 |\n",
      "========================= Model 23/288 =========================\n",
      "| Starting with model | 02_23_l-30_l-10_l-20_d-0.1 |\n",
      "| Starting time       | 2017-06-17 11:09:36.899496 |\n",
      "========================= Model 24/288 =========================\n",
      "| Starting with model | 02_24_l-30_l-10_l-20_d-0.2 |\n",
      "| Starting time       | 2017-06-17 11:17:40.313285 |\n",
      "========================= Model 25/288 =========================\n",
      "| Starting with model | 02_25_l-30_l-20            |\n",
      "| Starting time       | 2017-06-17 11:24:47.651955 |\n",
      "========================= Model 26/288 =========================\n",
      "| Starting with model | 02_26_l-30_l-20_d-0.1      |\n",
      "| Starting time       | 2017-06-17 11:28:18.776774 |\n",
      "========================= Model 27/288 =========================\n",
      "| Starting with model | 02_27_l-30_l-20_d-0.2      |\n",
      "| Starting time       | 2017-06-17 11:31:44.674811 |\n",
      "========================= Model 28/288 =========================\n",
      "| Starting with model | 02_28_l-30_l-20_l-10       |\n",
      "| Starting time       | 2017-06-17 11:36:23.430451 |\n",
      "========================= Model 29/288 =========================\n",
      "| Starting with model | 02_29_l-30_l-20_l-10_d-0.1 |\n",
      "| Starting time       | 2017-06-17 11:39:30.163149 |\n",
      "========================= Model 30/288 =========================\n",
      "| Starting with model | 02_30_l-30_l-20_l-10_d-0.2 |\n",
      "| Starting time       | 2017-06-17 11:44:37.302970 |\n",
      "========================= Model 31/288 =========================\n",
      "| Starting with model | 02_31_l-30_l-20_l-15       |\n",
      "| Starting time       | 2017-06-17 11:51:27.852108 |\n",
      "========================= Model 32/288 =========================\n",
      "| Starting with model | 02_32_l-30_l-20_l-15_d-0.1 |\n",
      "| Starting time       | 2017-06-17 11:55:44.193847 |\n",
      "========================= Model 33/288 =========================\n",
      "| Starting with model | 02_33_l-30_l-20_l-15_d-0.2 |\n",
      "| Starting time       | 2017-06-17 12:00:31.770607 |\n",
      "========================= Model 34/288 =========================\n",
      "| Starting with model | 02_34_l-30_l-20_l-20       |\n",
      "| Starting time       | 2017-06-17 12:10:51.045395 |\n",
      "========================= Model 35/288 =========================\n",
      "| Starting with model | 02_35_l-30_l-20_l-20_d-0.1 |\n",
      "| Starting time       | 2017-06-17 12:14:33.450349 |\n",
      "========================= Model 36/288 =========================\n",
      "| Starting with model | 02_36_l-30_l-20_l-20_d-0.2 |\n",
      "| Starting time       | 2017-06-17 12:20:39.264670 |\n",
      "========================= Model 37/288 =========================\n",
      "| Starting with model | 02_37_l-30_l-50            |\n",
      "| Starting time       | 2017-06-17 12:23:48.567334 |\n",
      "========================= Model 38/288 =========================\n",
      "| Starting with model | 02_38_l-30_l-50_d-0.1      |\n",
      "| Starting time       | 2017-06-17 12:29:38.466649 |\n",
      "========================= Model 39/288 =========================\n",
      "| Starting with model | 02_39_l-30_l-50_d-0.2      |\n",
      "| Starting time       | 2017-06-17 12:32:59.071624 |\n",
      "========================= Model 40/288 =========================\n",
      "| Starting with model | 02_40_l-30_l-50_l-10       |\n",
      "| Starting time       | 2017-06-17 12:39:48.406321 |\n",
      "========================= Model 41/288 =========================\n",
      "| Starting with model | 02_41_l-30_l-50_l-10_d-0.1 |\n",
      "| Starting time       | 2017-06-17 12:43:35.530372 |\n",
      "========================= Model 42/288 =========================\n",
      "| Starting with model | 02_42_l-30_l-50_l-10_d-0.2 |\n",
      "| Starting time       | 2017-06-17 12:50:55.766447 |\n",
      "========================= Model 43/288 =========================\n",
      "| Starting with model | 02_43_l-30_l-50_l-15       |\n",
      "| Starting time       | 2017-06-17 13:02:42.241269 |\n",
      "========================= Model 44/288 =========================\n",
      "| Starting with model | 02_44_l-30_l-50_l-15_d-0.1 |\n",
      "| Starting time       | 2017-06-17 13:11:57.656292 |\n",
      "========================= Model 45/288 =========================\n",
      "| Starting with model | 02_45_l-30_l-50_l-15_d-0.2 |\n",
      "| Starting time       | 2017-06-17 13:16:02.980068 |\n",
      "========================= Model 46/288 =========================\n",
      "| Starting with model | 02_46_l-30_l-50_l-20       |\n",
      "| Starting time       | 2017-06-17 13:19:18.245930 |\n",
      "========================= Model 47/288 =========================\n",
      "| Starting with model | 02_47_l-30_l-50_l-20_d-0.1 |\n",
      "| Starting time       | 2017-06-17 13:21:42.480973 |\n",
      "========================= Model 48/288 =========================\n",
      "| Starting with model | 02_48_l-30_l-50_l-20_d-0.2 |\n",
      "| Starting time       | 2017-06-17 13:27:43.962824 |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================= Model 49/288 =========================\n",
      "| Starting with model | 02_49_l-50                 |\n",
      "| Starting time       | 2017-06-17 13:32:09.201894 |\n",
      "========================= Model 50/288 =========================\n",
      "| Starting with model | 02_50_l-50_d-0.1           |\n",
      "| Starting time       | 2017-06-17 13:33:19.896017 |\n",
      "========================= Model 51/288 =========================\n",
      "| Starting with model | 02_51_l-50_d-0.2           |\n",
      "| Starting time       | 2017-06-17 13:35:43.594090 |\n",
      "========================= Model 52/288 =========================\n",
      "| Starting with model | 02_52_l-50_l-10            |\n",
      "| Starting time       | 2017-06-17 13:40:06.180392 |\n",
      "========================= Model 53/288 =========================\n",
      "| Starting with model | 02_53_l-50_l-10_d-0.1      |\n",
      "| Starting time       | 2017-06-17 13:45:10.797848 |\n",
      "========================= Model 54/288 =========================\n",
      "| Starting with model | 02_54_l-50_l-10_d-0.2      |\n",
      "| Starting time       | 2017-06-17 13:50:18.232172 |\n",
      "========================= Model 55/288 =========================\n",
      "| Starting with model | 02_55_l-50_l-15            |\n",
      "| Starting time       | 2017-06-17 13:57:17.940663 |\n",
      "========================= Model 56/288 =========================\n",
      "| Starting with model | 02_56_l-50_l-15_d-0.1      |\n",
      "| Starting time       | 2017-06-17 14:04:16.330594 |\n",
      "========================= Model 57/288 =========================\n",
      "| Starting with model | 02_57_l-50_l-15_d-0.2      |\n",
      "| Starting time       | 2017-06-17 14:09:39.166262 |\n",
      "========================= Model 58/288 =========================\n",
      "| Starting with model | 02_58_l-50_l-20            |\n",
      "| Starting time       | 2017-06-17 14:11:42.056698 |\n",
      "========================= Model 59/288 =========================\n",
      "| Starting with model | 02_59_l-50_l-20_d-0.1      |\n",
      "| Starting time       | 2017-06-17 14:14:34.483814 |\n",
      "========================= Model 60/288 =========================\n",
      "| Starting with model | 02_60_l-50_l-20_d-0.2      |\n",
      "| Starting time       | 2017-06-17 14:20:34.079223 |\n",
      "========================= Model 61/288 =========================\n",
      "| Starting with model | 02_61_l-50_l-10            |\n",
      "| Starting time       | 2017-06-17 14:30:49.963683 |\n",
      "========================= Model 62/288 =========================\n",
      "| Starting with model | 02_62_l-50_l-10_d-0.1      |\n",
      "| Starting time       | 2017-06-17 14:34:28.315732 |\n",
      "========================= Model 63/288 =========================\n",
      "| Starting with model | 02_63_l-50_l-10_d-0.2      |\n",
      "| Starting time       | 2017-06-17 14:41:54.141071 |\n",
      "========================= Model 64/288 =========================\n",
      "| Starting with model | 02_64_l-50_l-10_l-10       |\n",
      "| Starting time       | 2017-06-17 14:48:10.853911 |\n",
      "========================= Model 65/288 =========================\n",
      "| Starting with model | 02_65_l-50_l-10_l-10_d-0.1 |\n",
      "| Starting time       | 2017-06-17 14:51:26.079016 |\n",
      "========================= Model 66/288 =========================\n",
      "| Starting with model | 02_66_l-50_l-10_l-10_d-0.2 |\n",
      "| Starting time       | 2017-06-17 14:54:41.023126 |\n",
      "========================= Model 67/288 =========================\n",
      "| Starting with model | 02_67_l-50_l-10_l-15       |\n",
      "| Starting time       | 2017-06-17 15:00:25.447869 |\n",
      "========================= Model 68/288 =========================\n",
      "| Starting with model | 02_68_l-50_l-10_l-15_d-0.1 |\n",
      "| Starting time       | 2017-06-17 15:06:32.205912 |\n",
      "========================= Model 69/288 =========================\n",
      "| Starting with model | 02_69_l-50_l-10_l-15_d-0.2 |\n",
      "| Starting time       | 2017-06-17 15:16:32.523039 |\n",
      "========================= Model 70/288 =========================\n",
      "| Starting with model | 02_70_l-50_l-10_l-20       |\n",
      "| Starting time       | 2017-06-17 15:20:43.230922 |\n",
      "========================= Model 71/288 =========================\n",
      "| Starting with model | 02_71_l-50_l-10_l-20_d-0.1 |\n",
      "| Starting time       | 2017-06-17 15:24:08.136113 |\n",
      "========================= Model 72/288 =========================\n",
      "| Starting with model | 02_72_l-50_l-10_l-20_d-0.2 |\n",
      "| Starting time       | 2017-06-17 15:26:30.770105 |\n",
      "========================= Model 73/288 =========================\n",
      "| Starting with model | 02_73_l-50_l-20            |\n",
      "| Starting time       | 2017-06-17 15:41:29.623179 |\n",
      "========================= Model 74/288 =========================\n",
      "| Starting with model | 02_74_l-50_l-20_d-0.1      |\n",
      "| Starting time       | 2017-06-17 15:49:52.149372 |\n",
      "========================= Model 75/288 =========================\n",
      "| Starting with model | 02_75_l-50_l-20_d-0.2      |\n",
      "| Starting time       | 2017-06-17 15:51:39.941686 |\n",
      "========================= Model 76/288 =========================\n",
      "| Starting with model | 02_76_l-50_l-20_l-10       |\n",
      "| Starting time       | 2017-06-17 15:57:22.302235 |\n",
      "========================= Model 77/288 =========================\n",
      "| Starting with model | 02_77_l-50_l-20_l-10_d-0.1 |\n",
      "| Starting time       | 2017-06-17 15:59:58.618121 |\n",
      "========================= Model 78/288 =========================\n",
      "| Starting with model | 02_78_l-50_l-20_l-10_d-0.2 |\n",
      "| Starting time       | 2017-06-17 16:03:27.050579 |\n",
      "========================= Model 79/288 =========================\n",
      "| Starting with model | 02_79_l-50_l-20_l-15       |\n",
      "| Starting time       | 2017-06-17 16:06:52.265142 |\n",
      "========================= Model 80/288 =========================\n",
      "| Starting with model | 02_80_l-50_l-20_l-15_d-0.1 |\n",
      "| Starting time       | 2017-06-17 16:10:11.611277 |\n",
      "========================= Model 81/288 =========================\n",
      "| Starting with model | 02_81_l-50_l-20_l-15_d-0.2 |\n",
      "| Starting time       | 2017-06-17 16:13:26.929225 |\n",
      "========================= Model 82/288 =========================\n",
      "| Starting with model | 02_82_l-50_l-20_l-20       |\n",
      "| Starting time       | 2017-06-17 16:20:51.512234 |\n",
      "========================= Model 83/288 =========================\n",
      "| Starting with model | 02_83_l-50_l-20_l-20_d-0.1 |\n",
      "| Starting time       | 2017-06-17 16:24:13.530264 |\n",
      "========================= Model 84/288 =========================\n",
      "| Starting with model | 02_84_l-50_l-20_l-20_d-0.2 |\n",
      "| Starting time       | 2017-06-17 16:27:42.596336 |\n",
      "========================= Model 85/288 =========================\n",
      "| Starting with model | 02_85_l-50_l-50            |\n",
      "| Starting time       | 2017-06-17 16:31:04.644139 |\n",
      "========================= Model 86/288 =========================\n",
      "| Starting with model | 02_86_l-50_l-50_d-0.1      |\n",
      "| Starting time       | 2017-06-17 16:38:30.643691 |\n",
      "========================= Model 87/288 =========================\n",
      "| Starting with model | 02_87_l-50_l-50_d-0.2      |\n",
      "| Starting time       | 2017-06-17 16:47:11.049494 |\n",
      "========================= Model 88/288 =========================\n",
      "| Starting with model | 02_88_l-50_l-50_l-10       |\n",
      "| Starting time       | 2017-06-17 16:52:22.932045 |\n",
      "========================= Model 89/288 =========================\n",
      "| Starting with model | 02_89_l-50_l-50_l-10_d-0.1 |\n",
      "| Starting time       | 2017-06-17 16:55:13.796142 |\n",
      "========================= Model 90/288 =========================\n",
      "| Starting with model | 02_90_l-50_l-50_l-10_d-0.2 |\n",
      "| Starting time       | 2017-06-17 17:00:09.674513 |\n",
      "========================= Model 91/288 =========================\n",
      "| Starting with model | 02_91_l-50_l-50_l-15       |\n",
      "| Starting time       | 2017-06-17 17:07:24.509569 |\n",
      "========================= Model 92/288 =========================\n",
      "| Starting with model | 02_92_l-50_l-50_l-15_d-0.1 |\n",
      "| Starting time       | 2017-06-17 17:09:52.976379 |\n",
      "========================= Model 93/288 =========================\n",
      "| Starting with model | 02_93_l-50_l-50_l-15_d-0.2 |\n",
      "| Starting time       | 2017-06-17 17:14:47.848049 |\n",
      "========================= Model 94/288 =========================\n",
      "| Starting with model | 02_94_l-50_l-50_l-20       |\n",
      "| Starting time       | 2017-06-17 17:18:33.261907 |\n",
      "========================= Model 95/288 =========================\n",
      "| Starting with model | 02_95_l-50_l-50_l-20_d-0.1 |\n",
      "| Starting time       | 2017-06-17 17:22:35.589603 |\n",
      "========================= Model 96/288 =========================\n",
      "| Starting with model | 02_96_l-50_l-50_l-20_d-0.2 |\n",
      "| Starting time       | 2017-06-17 17:26:20.828251 |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================= Model 97/288 =========================\n",
      "| Starting with model | 02_97_l-75                 |\n",
      "| Starting time       | 2017-06-17 17:37:19.345712 |\n"
     ]
    }
   ],
   "source": [
    "start_time = t.time()\n",
    "for idx, m in enumerate(models):\n",
    "    stopper = t.time()\n",
    "    print('========================= Model {}/{} ========================='.format(idx+1, len(models)))\n",
    "    print(tabulate([['Starting with model', m['name']], ['Starting time', datetime.fromtimestamp(stopper)]],\n",
    "                   tablefmt=\"jira\", numalign=\"right\", floatfmt=\".3f\"))\n",
    "    try:\n",
    "        # Creating the Keras Model\n",
    "        model = lstm.create_model(layers=m['layers'], sample_size=X_train.shape[0], batch_size=m['batch_size'], \n",
    "                          timesteps=m['timesteps'], features=X_train.shape[1])\n",
    "        # Training...\n",
    "        history = lstm.train_model(model=model, mode='fit', y=y_train, X=X_train, \n",
    "                                   batch_size=m['batch_size'], timesteps=m['timesteps'], epochs=epochs, \n",
    "                                   rearrange=False, validation_split=validation_split, verbose=verbose, \n",
    "                                   early_stopping=early_stopping, min_delta=min_delta, patience=patience)\n",
    "\n",
    "        # Write results\n",
    "        min_loss = np.min(history.history['val_loss'])\n",
    "        min_idx = np.argmin(history.history['val_loss'])\n",
    "        min_epoch = min_idx + 1\n",
    "        \n",
    "        if verbose > 0:\n",
    "            print('______________________________________________________________________')\n",
    "            print(tabulate([['Minimum validation loss at epoch', min_epoch, 'Time: {}'.format(t.time()-stopper)],\n",
    "                        ['Training loss & MAE', history.history['loss'][min_idx], history.history['mean_absolute_error'][min_idx]  ], \n",
    "                        ['Validation loss & mae', history.history['val_loss'][min_idx], history.history['val_mean_absolute_error'][min_idx] ],\n",
    "                       ], tablefmt=\"jira\", numalign=\"right\", floatfmt=\".3f\"))\n",
    "            print('______________________________________________________________________')\n",
    "        \n",
    "        \n",
    "        result = [{'model_name': m['name'], 'config': m, 'train_loss': history.history['loss'][min_idx], 'train_rmse': 0,\n",
    "                   'train_mae': history.history['mean_absolute_error'][min_idx], 'train_mape': 0,\n",
    "                   'valid_loss': history.history['val_loss'][min_idx], 'valid_rmse': 0, \n",
    "                   'valid_mae': history.history['val_mean_absolute_error'][min_idx],'valid_mape': 0, \n",
    "                   'test_rmse': 0, 'test_mae': 0, 'test_mape': 0, 'epochs': '{}/{}'.format(min_epoch, epochs), 'batch_train':m['batch_size'],\n",
    "                   'input_shape':(X_train.shape[0], timesteps, X_train.shape[1]), 'total_time':t.time()-stopper, \n",
    "                   'time_step':0, 'splits':str(split_date), 'dropout': m['layers'][0]['dropout']\n",
    "                  }]\n",
    "        results = results.append(result, ignore_index=True)\n",
    "        \n",
    "        # Saving the model and weights\n",
    "        model.save(model_dir + m['name'] + '.h5')\n",
    "        \n",
    "        # Write results to csv\n",
    "        results.to_csv(output_table, sep=';')\n",
    "        \n",
    "        K.clear_session()\n",
    "        import tensorflow as tf\n",
    "        tf.reset_default_graph()\n",
    "        \n",
    "    # Shouldn't catch all errors, but for now...\n",
    "    except BaseException as e:\n",
    "        print('=============== ERROR {}/{} ============='.format(idx+1, len(models)))\n",
    "        print(tabulate([['Model:', m['name']], ['Config:', m]], tablefmt=\"jira\", numalign=\"right\", floatfmt=\".3f\"))\n",
    "        print('Error: {}'.format(e))\n",
    "        result = [{'model_name': m['name'], 'config': m, 'train_loss': str(e)}]\n",
    "        results = results.append(result, ignore_index=True)\n",
    "        results.to_csv(output_table,sep=';')\n",
    "        continue\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model selection based on the validation MAE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Select the top 5 models based on the Mean Absolute Error in the validation data:\n",
    "http://scikit-learn.org/stable/modules/model_evaluation.html#mean-absolute-error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of the selected top models \n",
    "selection = 5\n",
    "# If run in the same instance not necessary. If run on the same day, then just use output_table\n",
    "results_fn = res_dir + model_cat_id + '_results_' + '20170616' + '.csv'\n",
    "\n",
    "results_csv = pd.read_csv(results_fn, delimiter=';', encoding = 'latin1')\n",
    "top_models = results_csv.nsmallest(selection, 'valid_mae')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate top 5 models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Init test results table\n",
    "test_results = pd.DataFrame(columns=['Model name', 'Mean absolute error', 'Mean squared error'])\n",
    "\n",
    "# Init empty predictions\n",
    "predictions = {}\n",
    "\n",
    "# Loop through models\n",
    "for index, row in top_models.iterrows():\n",
    "    filename = model_dir + row['model_name'] + '.h5'\n",
    "    model = load_model(filename)\n",
    "    batch_size = int(row['batch_train'])\n",
    "    \n",
    "    # Calculate scores\n",
    "    loss, mae = lstm.evaluate_model(model=model, X=X_test, y=y_test, batch_size=batch_size, timesteps=1, verbose=verbose)\n",
    "    \n",
    "    # Store results\n",
    "    result = [{'Model name': row['model_name'], \n",
    "               'Mean squared error': loss, 'Mean absolute error': mae\n",
    "              }]\n",
    "    test_results = test_results.append(result, ignore_index=True)\n",
    "    \n",
    "    # Generate predictions\n",
    "    model.reset_states()\n",
    "    model_predictions = lstm.get_predictions(model=model, X=X_test, batch_size=batch_size, timesteps=timesteps[0], verbose=verbose)\n",
    "    \n",
    "    # Save predictions\n",
    "    predictions[row['model_name']] = model_predictions\n",
    "    \n",
    "    K.clear_session()\n",
    "    import tensorflow as tf\n",
    "    tf.reset_default_graph()\n",
    "    \n",
    "\n",
    "test_results = test_results.sort_values('Mean absolute error', ascending=True)\n",
    "test_results = test_results.set_index(['Model name'])\n",
    "\n",
    "if not os.path.isfile(test_output_table):\n",
    "    test_results.to_csv(test_output_table, sep=';')\n",
    "else: # else it exists so append without writing the header\n",
    "    test_results.to_csv(test_output_table,mode = 'a',header=False, sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test dataset performance of the best 5 (out of 432 tested models):\n",
      "+--------------------------+-----------------------+----------------------+\n",
      "| Model name               |   Mean absolute error |   Mean squared error |\n",
      "+==========================+=======================+======================+\n",
      "| 1_02_2_l-5_d-0.1         |                 0.305 |                0.151 |\n",
      "+--------------------------+-----------------------+----------------------+\n",
      "| 1_02_50_l-10_d-0.1       |                 0.320 |                0.165 |\n",
      "+--------------------------+-----------------------+----------------------+\n",
      "| 1_02_135_l-20_l-50_d-0.2 |                 0.341 |                0.181 |\n",
      "+--------------------------+-----------------------+----------------------+\n",
      "| 3_02_51_l-100_d-0.2      |                 0.351 |                0.189 |\n",
      "+--------------------------+-----------------------+----------------------+\n",
      "| 3_02_49_l-100            |                 0.367 |                0.209 |\n",
      "+--------------------------+-----------------------+----------------------+\n"
     ]
    }
   ],
   "source": [
    "print('Test dataset performance of the best {} (out of {} tested models):'.format(min(selection, len(models)), len(models)))\n",
    "print(tabulate(test_results, headers='keys', tablefmt=\"grid\", numalign=\"right\", floatfmt=\".3f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}