Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Setup configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import pandas as pds\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "\n",
    "from argparse import ArgumentParser\n",
    "import gc ; gc.enable()\n",
    "import torch\n",
    "from torch import nn\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from torchmetrics import Accuracy,MeanAbsolutePercentageError\n",
    "from pytorch_forecasting import SMAPE\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "import joblib\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n",
    "from src.config import load_default, dataset_name_check, setup_config_with_method\n",
    "\n",
    "\n",
    "config = load_default()\n",
    "dataset_name_check(config,config.dataset_name)\n",
    "setup_config_with_method(config)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    n_gpu = torch.cuda.device_count()\n",
    "    config.gpu_numb = [n_gpu-1]\n",
    "\n",
    "print(\"CUDA Available:\", torch.cuda.is_available())\n",
    "print(\"Number of GPUs:\", torch.cuda.device_count())\n",
    "print(\"GPU Number Config:\", config.gpu_numb)\n",
    "\n",
    "device = torch.device('cuda:{}'.format(config.gpu_numb[0])) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%\n",
    "raw_df = pds.read_csv(config.data_path)\n",
    "\n",
    "if 'sample_data' in config.data_path:\n",
    "    raw_df = raw_df.drop(columns = 'Date')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Standart Scaling & Preparing Train-Validation Size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%\n",
    "scaler = StandardScaler()\n",
    "scaled_df = scaler.fit_transform(raw_df)\n",
    "\n",
    "nsamples = len(raw_df)\n",
    "train_size = int(nsamples * config.training_size)\n",
    "val_size = int(nsamples * 0.1)\n",
    "\n",
    "\n",
    "# Custom split generator\n",
    "def custom_time_series_split(data, n_splits, train_size, val_size):\n",
    "    total_size = train_size + val_size\n",
    "    indices = np.arange(len(data))\n",
    "    splits = []\n",
    "    step = (len(data) - total_size) // (n_splits - 1)\n",
    "    for i in range(n_splits):\n",
    "        start = i * step\n",
    "        end = start + total_size\n",
    "        if end <= len(data):\n",
    "            train_indices = indices[start:start + train_size]\n",
    "            val_indices = indices[start + train_size:start + total_size]\n",
    "            splits.append((train_indices, val_indices))\n",
    "    return splits\n",
    "\n",
    "splits = custom_time_series_split(scaled_df, config.n_fold, train_size, val_size)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Defining Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%\n",
    "def fn_smape(y, y_pred):\n",
    "    return ((2 * np.abs(y - y_pred)) / (np.abs(y) + np.abs(y_pred))).mean()\n",
    "\n",
    "def compute_metric(y_true, y_pred):\n",
    "    metrics = {\n",
    "        'R2': r2_score(y_true, y_pred),\n",
    "        'RMSE': mean_squared_error(y_true, y_pred) ** 0.5,\n",
    "        'MAE': mean_absolute_error(y_true, y_pred),\n",
    "        'SMAPE': fn_smape(y_true, y_pred)\n",
    "    }\n",
    "    return metrics\n",
    "\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Trainig & Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%\n",
    "# Loop for Cross-Validation\n",
    "\n",
    "for fold, (train_index, val_index) in enumerate(splits):   \n",
    "    print(f\"Training fold {fold + 1}/{config.n_fold}\")\n",
    "    \n",
    "    train_df = scaled_df[train_index]\n",
    "    valid_df = scaled_df[val_index]\n",
    "\n",
    "    print(f\"Fold {fold + 1} - Training indices: {train_index[:5]}...{train_index[-5:]}\")\n",
    "    print(f\"Fold {fold + 1} - Validation indices: {val_index[:5]}...{val_index[-5:]}\")\n",
    "    \n",
    "    # Sequences Slicing \n",
    "    train_seqs = np.lib.stride_tricks.sliding_window_view(\n",
    "        x=train_df,\n",
    "        window_shape=(config.input_length + config.output_length),\n",
    "        axis=0\n",
    "    ).transpose([0, 2, 1])\n",
    "    \n",
    "    valid_seqs = np.lib.stride_tricks.sliding_window_view(\n",
    "        x=valid_df,\n",
    "        window_shape=(config.input_length + config.output_length),\n",
    "        axis=0\n",
    "    ).transpose([0, 2, 1])\n",
    "    \n",
    "    dataset_dict = dict(\n",
    "        train=(train_seqs[:, :-1], train_seqs[:, -1]),\n",
    "        valid=(valid_seqs[:, :-1], valid_seqs[:, -1])\n",
    "    )\n",
    "    \n",
    "    from src.models.model_selector import get_model\n",
    "    from src.data_prepare import pl_DataModule\n",
    "    \n",
    "    # Preparing the model \n",
    "    pldm = pl_DataModule(dataset_dict, config) \n",
    "    \n",
    "    config.input_feature = raw_df.shape[1]\n",
    "    config.output_feature = raw_df.shape[1]\n",
    "    \n",
    "    model = get_model(config) # Model Selection NATM (Feature, Time, Independent), DNN, LSTM, SCINet\n",
    "    \n",
    "    model.train()\n",
    "    save_name = '_'.join([config.exp_name, config.method, config.dataset_name, str(config.input_length)])\n",
    "    \n",
    "    callbacks = [\n",
    "        ModelCheckpoint(\n",
    "            dirpath=os.path.join('.', config.save_ckpt_dirs, save_name, f'fold_{fold + 1}'),\n",
    "            filename='{epoch:03d}-{val_loss:.3f}-{val_SMAPE:.3f}',\n",
    "            save_last=True,\n",
    "            save_top_k=config.save_top_k,\n",
    "            monitor='val_loss',\n",
    "        ),\n",
    "        EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=config.ealry_stop_round,\n",
    "        )\n",
    "    ]\n",
    "    \n",
    "    trainer = pl.Trainer(\n",
    "        accelerator='gpu' if torch.cuda.is_available() else 'cpu',\n",
    "        enable_progress_bar=config.prog_bar,\n",
    "        devices=config.gpu_numb,\n",
    "        max_epochs=config.epochs,\n",
    "        callbacks=callbacks,\n",
    "    )\n",
    "    \n",
    "    # Count the number of parameters\n",
    "    num_params = count_parameters(model)\n",
    "    print(f\"Fold {fold + 1} - Number of parameters: {num_params}\")\n",
    "\n",
    "    # Measure the training time\n",
    "    start_time = time.time()\n",
    "\n",
    "    # Training \n",
    "    trainer.fit(model, datamodule=pldm)\n",
    "\n",
    "    end_time = time.time()\n",
    "    training_time = end_time - start_time\n",
    "    print(f\"Fold {fold + 1} - Training time: {training_time:.2f} seconds\")\n",
    "    \n",
    "    # Logging\n",
    "    model.eval()\n",
    "    outputs = trainer.predict(model, pldm.val_dataloader())\n",
    "    \n",
    "    return_trues = []\n",
    "    return_preds = []\n",
    "    \n",
    "    for output in outputs:\n",
    "        if len(output) == 4:\n",
    "            yt, yp, met, w = output\n",
    "        else:\n",
    "            yt, yp, w = output\n",
    "        \n",
    "        return_trues.append(yt.numpy())\n",
    "        return_preds.append(yp.numpy())\n",
    "    \n",
    "    return_trues = scaler.inverse_transform(np.concatenate(return_trues))\n",
    "    return_preds = scaler.inverse_transform(np.concatenate(return_preds))\n",
    "    \n",
    "    joblib.dump(\n",
    "    dict(\n",
    "        scaler = scaler,\n",
    "        config = config,\n",
    "        columns = raw_df.columns\n",
    "    ), os.path.join('.',config.save_ckpt_dirs, save_name, f'fold_{fold + 1}', 'log.joblib')\n",
    "    )\n",
    "\n",
    "    joblib.dump(\n",
    "    dict(\n",
    "        train = train_seqs,\n",
    "        valid = valid_seqs,\n",
    "    ), os.path.join('.',config.save_ckpt_dirs, save_name, f'fold_{fold + 1}', 'data_samples.joblib')\n",
    "    )\n",
    "\n",
    "    # Save train and validation indices\n",
    "    indices_path = os.path.join('.', config.save_ckpt_dirs, save_name, f'fold_{fold + 1}', 'indices.joblib')\n",
    "    joblib.dump({'train_index': train_index, 'val_index': val_index}, indices_path)\n",
    "    \n",
    "    # Extracting Price Feature\n",
    "    feature_true = return_trues[:, 0]\n",
    "    feature_pred = return_preds[:, 0]\n",
    "    \n",
    "    #Print sizes of feature_true and feature_pred for inspection\n",
    "    print(f\"Fold {fold + 1} - feature_true size: {feature_true.shape}, feature_pred size: {feature_pred.shape}\")\n",
    "    \n",
    "    # Evaluation\n",
    "    metrics = compute_metric(feature_true, feature_pred)\n",
    "    metrics.update({\n",
    "        'method': config.method,\n",
    "        'input_length': config.input_length,\n",
    "        'dataset_name': config.dataset_name,\n",
    "        'fold': fold + 1,\n",
    "        'num_params': num_params,\n",
    "        'training_time': training_time\n",
    "    })\n",
    "\n",
    "\n",
    "    df_metrics = pds.DataFrame([metrics])\n",
    "\n",
    "    results_dir = 'results'\n",
    "    csv_path = os.path.join(results_dir, 'evaluation_metrics_results.csv')\n",
    "\n",
    "    os.makedirs(results_dir, exist_ok=True)\n",
    "    print(\"Writing to CSV now...\") # Controlling\n",
    "    print(df_metrics) # Controlling\n",
    "\n",
    "    if os.path.isfile(csv_path):\n",
    "        df_metrics.to_csv(csv_path, mode='a', header=False, index=False)\n",
    "    else:\n",
    "        df_metrics.to_csv(csv_path, mode='w', header=True, index=False)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}