Run_Expression_Classifier_on_Webcam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "Run_Expression_Classifier_on_Webcam.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "3jy1XEVj51Th",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys, os\n",
        "import numpy as np\n",
        "import os\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "import torchvision\n",
        "from PIL import Image\n",
        "import torch.nn.functional as F\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from tqdm import tnrange, tqdm_notebook, tqdm\n",
        "from IPython.display import clear_output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RN-lCPKfDbNY",
        "colab_type": "text"
      },
      "source": [
        "### Load the model and define a list of classes.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lm76FLQ851Ty",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "emotions = ['disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral', 'angry']\n",
        "\n",
        "def get_num_correct(preds, labels):\n",
        "    return preds.argmax(dim=1).eq(labels).sum().item()\n",
        "\n",
        "class Network(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Network, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=5, stride=1, padding=0,)\n",
        "        self.pool = nn.MaxPool2d(2, 2)\n",
        "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
        "        self.fc1 = nn.Linear(16 * 9 * 9, 300)\n",
        "        self.fc2 = nn.Linear(300, 60)\n",
        "        self.fc3 = nn.Linear(60, 7)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = self.pool(F.relu(self.conv1(x)))\n",
        "        x = self.pool(F.relu(self.conv2(x)))\n",
        "        x = x.view(-1, 16 * 9 * 9)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return x\n",
        "\n",
        "network=Network()\n",
        "network.load_state_dict(torch.load('./model/model_f2.pt', map_location=torch.device('cpu')))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kwvk5_w_CTFd",
        "colab_type": "text"
      },
      "source": [
        "### Hard-code warning!\n",
        "The display_emoji function creates a CV2 window to display an image of the emoji corresponding to the predicted class.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b1V5rnCe51UD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def display_emoji(max_index):\n",
        "    if max_index == 0:\n",
        "        path = './emojis/disguist.png'\n",
        "        im = cv2.imread(path)\n",
        "        cv2.imshow('', im)\n",
        "\n",
        "    if max_index == 1:\n",
        "        path = './emojis/fear.png'\n",
        "        im = cv2.imread(path)\n",
        "        cv2.imshow('', im)\n",
        "        \n",
        "    if max_index == 2:\n",
        "        path = './emojis/happy.png'\n",
        "        im = cv2.imread(path)\n",
        "        cv2.imshow('', im)\n",
        "\n",
        "    if max_index == 3:\n",
        "        path = './emojis/sad.png'\n",
        "        im = cv2.imread(path)\n",
        "        cv2.imshow('', im)\n",
        "\n",
        "    if max_index == 4:\n",
        "        path = './emojis/surprise.png'\n",
        "        im = cv2.imread(path)\n",
        "        cv2.imshow('', im)\n",
        "        \n",
        "    if max_index == 5:\n",
        "        path = './emojis/neutral.png'\n",
        "        im = cv2.imread(path)\n",
        "        cv2.imshow('', im)\n",
        "\n",
        "    if max_index == 6:\n",
        "        path = './emojis/angry.png'\n",
        "        im = cv2.imread(path)\n",
        "        cv2.imshow('', im)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GKD7Oa50Crj4",
        "colab_type": "text"
      },
      "source": [
        "### End Game:\n",
        "The cell below streams webcam frames through the trained model, which outputs a tensor. Then display_emoji displays the corresponding image. \n",
        "\n",
        "### Note :\n",
        "Press 'Q' to terminate."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GlOZFH5651UP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "face_haar_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')\n",
        "cap=cv2.VideoCapture(0)\n",
        "\n",
        "\n",
        "while True:\n",
        "    \n",
        "    ret,test_img=cap.read()# captures frame and returns boolean value and captured image\n",
        "    if not ret:\n",
        "        continue\n",
        "    gray_img= cv2.cvtColor(test_img, cv2.COLOR_BGR2GRAY)\n",
        "    \n",
        "    faces_detected = face_haar_cascade.detectMultiScale(gray_img, 1.32, 5)\n",
        "        \n",
        "    for (x,y,w,h) in faces_detected:\n",
        "        \n",
        "        cv2.rectangle(test_img,(x,y),(x+w,y+h),(255,0,0),thickness=7)\n",
        "        roi_gray=gray_img[y:y+w,x:x+h]#cropping region of interest i.e. face area from  image\n",
        "        roi_gray=cv2.resize(roi_gray,(48,48))\n",
        "        img_pixels = (roi_gray)\n",
        "        img_pixels = np.expand_dims(img_pixels, axis = 0)\n",
        "        p_image=torch.Tensor(img_pixels)\n",
        "        img_pixels //= 255\n",
        "        img_pixels=torch.Tensor(img_pixels)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            \n",
        "            clear_output(wait=True)\n",
        "            network.eval()\n",
        "            #p_image=p_image.cuda()\n",
        "            lab=network(p_image.unsqueeze(0))\n",
        "            #lab=lab.cpu()\n",
        "            max_index = np.argmax(lab[0])\n",
        "            predicted_emotion = emotions[max_index] \n",
        "            display_emoji(max_index)\n",
        "            cv2.putText(test_img, predicted_emotion, (int(x), int(y)), cv2.FONT_HERSHEY_PLAIN, 1, (0,0,255), 2)\n",
        "            #plt.plot(lab[0])\n",
        "\n",
        "    resized_img = cv2.resize(test_img, (700, 500))\n",
        "    cv2.imshow('Facial-expression-clasifier',resized_img)\n",
        "\n",
        "    if cv2.waitKey(10) == ord('q'):\n",
        "        cv2.destroyAllWindows()\n",
        "        break"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}