Show results JAAD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting the JAAD images from videos and showing the crossing prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from torch import from_numpy\n",
    "from torch import cuda\n",
    "from torch import no_grad\n",
    "from torch import optim\n",
    "\n",
    "from torch_geometric.data import Data\n",
    "from torch_geometric.loader import DataLoader\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.figure import Figure\n",
    "from matplotlib.backends.backend_agg import FigureCanvasAgg\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "from Code.GNN import *\n",
    "from Code.SkeletonsDataset import *\n",
    "from Code.ModelTrainEvaluate import *\n",
    "\n",
    "import cv2\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the skeletons dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasetName = 'JAAD'\n",
    "subset = 'test'\n",
    "poseEstimator = 'AlphaPose'\n",
    "numberOfClasses = 2\n",
    "info = 87\n",
    "data_augm = '' #'dataAugmentation4'\n",
    "\n",
    "net = 'TGCN'\n",
    "dropout = 0.5\n",
    "num_epochs = 1000\n",
    "batch_size = 10000\n",
    "\n",
    "data_augmentation_name = '' if data_augm == None or data_augm == '' else '_' + data_augm\n",
    "numberOfJoints = 25 if poseEstimator == 'OpenPose' else 18\n",
    "\n",
    "dataset = SkeletonsDataset('Data/' + datasetName + '/' + subset + '_jaad_' + poseEstimator + '.csv',\n",
    "                           normalization='minmax', target='cross', info=info,\n",
    "                           remove_undetected=True, numberOfJoints=numberOfJoints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['video_0005',\n",
       " 'video_0016',\n",
       " 'video_0017',\n",
       " 'video_0028',\n",
       " 'video_0042',\n",
       " 'video_0045',\n",
       " 'video_0046',\n",
       " 'video_0048',\n",
       " 'video_0053',\n",
       " 'video_0055',\n",
       " 'video_0059',\n",
       " 'video_0071',\n",
       " 'video_0076',\n",
       " 'video_0084',\n",
       " 'video_0087',\n",
       " 'video_0090',\n",
       " 'video_0092',\n",
       " 'video_0093',\n",
       " 'video_0096',\n",
       " 'video_0097',\n",
       " 'video_0100',\n",
       " 'video_0101',\n",
       " 'video_0103',\n",
       " 'video_0104',\n",
       " 'video_0105',\n",
       " 'video_0106',\n",
       " 'video_0107',\n",
       " 'video_0110',\n",
       " 'video_0113',\n",
       " 'video_0115',\n",
       " 'video_0116',\n",
       " 'video_0117',\n",
       " 'video_0118',\n",
       " 'video_0124',\n",
       " 'video_0125',\n",
       " 'video_0128',\n",
       " 'video_0135',\n",
       " 'video_0141',\n",
       " 'video_0144',\n",
       " 'video_0148',\n",
       " 'video_0150',\n",
       " 'video_0151',\n",
       " 'video_0152',\n",
       " 'video_0155',\n",
       " 'video_0162',\n",
       " 'video_0163',\n",
       " 'video_0164',\n",
       " 'video_0165',\n",
       " 'video_0173',\n",
       " 'video_0177',\n",
       " 'video_0178',\n",
       " 'video_0179',\n",
       " 'video_0183',\n",
       " 'video_0187',\n",
       " 'video_0197',\n",
       " 'video_0201',\n",
       " 'video_0203',\n",
       " 'video_0206',\n",
       " 'video_0211',\n",
       " 'video_0212',\n",
       " 'video_0213',\n",
       " 'video_0216',\n",
       " 'video_0221',\n",
       " 'video_0222',\n",
       " 'video_0223',\n",
       " 'video_0224',\n",
       " 'video_0230',\n",
       " 'video_0234',\n",
       " 'video_0238',\n",
       " 'video_0239',\n",
       " 'video_0243',\n",
       " 'video_0244',\n",
       " 'video_0245',\n",
       " 'video_0251',\n",
       " 'video_0253',\n",
       " 'video_0265',\n",
       " 'video_0267',\n",
       " 'video_0270',\n",
       " 'video_0271',\n",
       " 'video_0277',\n",
       " 'video_0278',\n",
       " 'video_0279',\n",
       " 'video_0280',\n",
       " 'video_0285',\n",
       " 'video_0287',\n",
       " 'video_0288',\n",
       " 'video_0292',\n",
       " 'video_0294',\n",
       " 'video_0295',\n",
       " 'video_0299',\n",
       " 'video_0300',\n",
       " 'video_0304',\n",
       " 'video_0305',\n",
       " 'video_0307',\n",
       " 'video_0308',\n",
       " 'video_0309',\n",
       " 'video_0313',\n",
       " 'video_0314',\n",
       " 'video_0316',\n",
       " 'video_0322',\n",
       " 'video_0327',\n",
       " 'video_0329',\n",
       " 'video_0330',\n",
       " 'video_0332',\n",
       " 'video_0333',\n",
       " 'video_0334',\n",
       " 'video_0336',\n",
       " 'video_0337',\n",
       " 'video_0338',\n",
       " 'video_0339',\n",
       " 'video_0344']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "videos_list = dataset.loadedData['video'].unique().tolist()\n",
    "\n",
    "videos_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video</th>\n",
       "      <th>frame</th>\n",
       "      <th>skeleton</th>\n",
       "      <th>cross</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>video_0005</td>\n",
       "      <td>0</td>\n",
       "      <td>[[998.39892578125, 694.7347412109375, 0.937213...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>video_0005</td>\n",
       "      <td>1</td>\n",
       "      <td>[[998.4149169921875, 694.6920166015625, 0.9358...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>video_0005</td>\n",
       "      <td>2</td>\n",
       "      <td>[[998.3433227539062, 696.0174560546875, 0.9455...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>video_0005</td>\n",
       "      <td>3</td>\n",
       "      <td>[[997.6259765625, 696.2772216796875, 0.9139388...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>video_0005</td>\n",
       "      <td>4</td>\n",
       "      <td>[[996.3665161132812, 697.2818603515625, 0.9003...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52961</th>\n",
       "      <td>video_0344</td>\n",
       "      <td>85</td>\n",
       "      <td>[[1767.3865966796875, 764.985107421875, 0.9258...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52962</th>\n",
       "      <td>video_0344</td>\n",
       "      <td>86</td>\n",
       "      <td>[[1785.76220703125, 762.012451171875, 0.931533...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52963</th>\n",
       "      <td>video_0344</td>\n",
       "      <td>87</td>\n",
       "      <td>[[1807.765625, 760.1240234375, 0.8868738412857...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52964</th>\n",
       "      <td>video_0344</td>\n",
       "      <td>88</td>\n",
       "      <td>[[1858.822021484375, 766.7747802734375, 0.9220...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52965</th>\n",
       "      <td>video_0344</td>\n",
       "      <td>89</td>\n",
       "      <td>[[1884.9903564453125, 771.9426879882812, 0.915...</td>\n",
       "      <td>not-crossing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>52485 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            video  frame                                           skeleton  \\\n",
       "0      video_0005      0  [[998.39892578125, 694.7347412109375, 0.937213...   \n",
       "1      video_0005      1  [[998.4149169921875, 694.6920166015625, 0.9358...   \n",
       "2      video_0005      2  [[998.3433227539062, 696.0174560546875, 0.9455...   \n",
       "3      video_0005      3  [[997.6259765625, 696.2772216796875, 0.9139388...   \n",
       "4      video_0005      4  [[996.3665161132812, 697.2818603515625, 0.9003...   \n",
       "...           ...    ...                                                ...   \n",
       "52961  video_0344     85  [[1767.3865966796875, 764.985107421875, 0.9258...   \n",
       "52962  video_0344     86  [[1785.76220703125, 762.012451171875, 0.931533...   \n",
       "52963  video_0344     87  [[1807.765625, 760.1240234375, 0.8868738412857...   \n",
       "52964  video_0344     88  [[1858.822021484375, 766.7747802734375, 0.9220...   \n",
       "52965  video_0344     89  [[1884.9903564453125, 771.9426879882812, 0.915...   \n",
       "\n",
       "              cross  \n",
       "0      not-crossing  \n",
       "1      not-crossing  \n",
       "2      not-crossing  \n",
       "3      not-crossing  \n",
       "4      not-crossing  \n",
       "...             ...  \n",
       "52961  not-crossing  \n",
       "52962  not-crossing  \n",
       "52963  not-crossing  \n",
       "52964  not-crossing  \n",
       "52965  not-crossing  \n",
       "\n",
       "[52485 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.loadedData[['video','frame','skeleton','cross']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of nodes per skeleton: 18\n",
      "Number of features per node: 3\n"
     ]
    }
   ],
   "source": [
    "# First element of the dataset:\n",
    "t0 = dataset[0]\n",
    "\n",
    "# Node features:\n",
    "t1 = t0.x_temporal[0]\n",
    "\n",
    "# Number of nodes:\n",
    "numberOfNodes = t1.shape[0]\n",
    "\n",
    "# Number of dimensions of each node features:\n",
    "embed_dim = t1.shape[1]\n",
    "\n",
    "print('Number of nodes per skeleton:', numberOfNodes)\n",
    "print('Number of features per node:', embed_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model: TGCN_AlphaPose_info=87_dropout=0.5_epoch=1000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modelName = net + '_' + poseEstimator + '_info=' + str(info) + '_dropout=' + str(dropout) + data_augmentation_name\n",
    "modelName = modelName + '_epoch=' + str(num_epochs)\n",
    "\n",
    "print('Loading model:', modelName)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "model = SpatialTemporalGNN(embed_dim, numberOfClasses, numberOfNodes, net=net,\n",
    "                           filterSize=embed_dim, dropout=dropout, batchSize=batch_size).to(device)\n",
    "\n",
    "model.load_state_dict(torch.load('exportedModels/' + datasetName + '/' + modelName))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making the crossing/not-crossing prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = DataLoader(dataset, batch_size=batch_size)\n",
    "\n",
    "predictions, groundtruth = predict(model, loader, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the video and exporting the result as a GIF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting processing of video video_0005\n",
      "Exported video: video_0005  - Clip 1/10 \n",
      "\n",
      "Starting processing of video video_0016\n",
      "Exported video: video_0016  - Clip 2/10 \n",
      "\n",
      "Starting processing of video video_0017\n",
      "Exported video: video_0017  - Clip 3/10 \n",
      "\n",
      "Starting processing of video video_0028\n",
      "Exported video: video_0028  - Clip 4/10 \n",
      "\n",
      "Starting processing of video video_0042\n",
      "Exported video: video_0042  - Clip 5/10 \n",
      "\n",
      "Starting processing of video video_0045\n",
      "Exported video: video_0045  - Clip 6/10 \n",
      "\n",
      "Starting processing of video video_0046\n",
      "Exported video: video_0046  - Clip 7/10 \n",
      "\n",
      "Starting processing of video video_0048\n",
      "Exported video: video_0048  - Clip 8/10 \n",
      "\n",
      "Starting processing of video video_0053\n",
      "Exported video: video_0053  - Clip 9/10 \n",
      "\n",
      "Starting processing of video video_0055\n",
      "Exported video: video_0055  - Clip 10/10 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "os.makedirs(\"Videos_results/\" + datasetName + '/' + poseEstimator + \"/\" + subset + \"/\" + modelName, exist_ok=True)\n",
    "\n",
    "quantity = 10\n",
    "\n",
    "gen_clips = os.listdir(\"Data/\" + datasetName + \"-videos/\")\n",
    "\n",
    "videos_list_frames = dataset.loadedData['video'].tolist()\n",
    "\n",
    "exported = 0\n",
    "\n",
    "for video_id in videos_list[0:10]:\n",
    "    \n",
    "    print('Starting processing of video', video_id)\n",
    "\n",
    "    video = cv2.VideoCapture(\"Data/\" + datasetName + \"-videos/\" + video_id + \".mp4\")\n",
    "\n",
    "\n",
    "    # First column in the dataset where the video starts:\n",
    "    video_first_dataset_row = videos_list_frames.index(video_id)\n",
    "    \n",
    "    frames = dataset.loadedData[dataset.loadedData['video'] == video_id].sort_values(by=['frame', 'ped_id'], ascending=True)\n",
    "    frames = frames['frame'].tolist()\n",
    "    frames_rev = frames[::-1]\n",
    "\n",
    "    video_outputs = []\n",
    "\n",
    "    frame_i = 0\n",
    "    ret = True\n",
    "    while ret:\n",
    "\n",
    "        ret, frame = video.read()\n",
    "\n",
    "        if ret:\n",
    "            \n",
    "            frame = frame[...,::-1]\n",
    "            \n",
    "            if frame_i in frames:\n",
    "            \n",
    "                frame_first_index = frames.index(frame_i)\n",
    "                frame_last_index = len(frames_rev) - frames_rev.index(frame_i) - 1\n",
    "\n",
    "                frame_prediction = predictions[video_first_dataset_row + frame_first_index:video_first_dataset_row + frame_last_index + 1]\n",
    "                frame_groundtruth = groundtruth[video_first_dataset_row + frame_first_index:video_first_dataset_row + frame_last_index + 1]\n",
    "\n",
    "                frame_prediction = [\"Crossing\" if f else \"Not-crossing\" for f in frame_prediction]\n",
    "                frame_groundtruth = [\"Crossing\" if f else \"Not-crossing\" for f in frame_groundtruth]\n",
    "\n",
    "                #im_title = \"Prediction: \" + frame_prediction + \"\\nGroundtruth: \" + frame_groundtruth\n",
    "                im_title = \"Video: \" + video_id\n",
    "                im_title = im_title + \" - Frame: \" + str(frame_i)\n",
    "                im_title = im_title + \"\\nPose estimator: \" + poseEstimator #if frame_i == 0 else \"\"\n",
    "                im_title = im_title + \"\\nClassifier: \" + net + '_info=' + str(info) + '_dropout=' + str(dropout) + data_augmentation_name + '_epoch=' + str(num_epochs)\n",
    "\n",
    "\n",
    "                fig = dataset.showSkeleton(videoNum=video_id, frameNum=frame_i, showLegend=False, frameImage=frame,\n",
    "                                           normalizedSkeletons=False, title=im_title, show=False,\n",
    "                                           predictions=frame_prediction, groundtruths=frame_groundtruth)\n",
    "\n",
    "\n",
    "                canvas = FigureCanvasAgg(fig)\n",
    "                canvas.draw()\n",
    "                frame_result = np.asarray(canvas.buffer_rgba()).astype(np.uint8)\n",
    "\n",
    "                frame_result = Image.fromarray(frame_result)\n",
    "\n",
    "                video_outputs.append(frame_result)\n",
    "\n",
    "                canvas.get_renderer().clear()\n",
    "                plt.close(fig)\n",
    "\n",
    "        frame_i = frame_i + 1\n",
    "                \n",
    "\n",
    "    \n",
    "    # Export the prediction result as a GIF:\n",
    "    if len(video_outputs) > 0:\n",
    "        video_outputs[0].save(\"Videos_results/\" + datasetName + '/' + poseEstimator + \"/\" + subset + \"/\" + modelName + \"/\" + video_id + \".gif\", \n",
    "                              save_all=True, append_images=video_outputs[1:], duration=30, loop=0, disposal=2)\n",
    "    \n",
    "    exported = exported + 1\n",
    "    \n",
    "    print('Exported video:', video_id, ' - Clip', str(exported) + '/' + str(quantity), '\\n')\n",
    "    \n",
    "    video.release()\n",
    "    \n",
    "    if exported == quantity:\n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc-autonumbering": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}