DCNN_FERPlus.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "DCNN_FERPlus.ipynb",
      "provenance": [],
      "toc_visible": true,
      "mount_file_id": "1AtXRjOm_-l5pCm8NP7WxuYVQGBdQB7a7",
      "authorship_tag": "ABX9TyNuNkmxKXRwcgnji977ft9b",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DreamWalker1412/Colab/blob/main/DCNN_FERPlus.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NT1wev-HDR-J"
      },
      "source": [
        "# download cntk & data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L7qbguDDJJx4"
      },
      "source": [
        "!apt-get install --no-install-recommends openmpi-bin libopenmpi-dev libopencv-dev python3-opencv python-opencv && ln -sf /usr/lib/x86_64-linux-gnu/libmpi_cxx.so /usr/lib/x86_64-linux-gnu/libmpi_cxx.so.1 && ln -sf /usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so /usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.12 && ln -sf /usr/lib/x86_64-linux-gnu/libmpi.so /usr/lib/x86_64-linux-gnu/libmpi.so.12 && pip install cntk-gpu"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "esJdnxbeEIWx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "899c031d-c15d-4418-c235-4caf7b8d7aa7"
      },
      "source": [
        "!git clone https://github.com/microsoft/FERPlus.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'FERPlus'...\n",
            "remote: Enumerating objects: 128, done.\u001b[K\n",
            "remote: Total 128 (delta 0), reused 0 (delta 0), pack-reused 128\u001b[K\n",
            "Receiving objects: 100% (128/128), 650.37 KiB | 13.84 MiB/s, done.\n",
            "Resolving deltas: 100% (70/70), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_l4kgw5TDIW3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff9215eb-bca9-4766-f319-a0ddc54192bb"
      },
      "source": [
        "!unzip -n /content/drive/MyDrive/data/fer2013.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/drive/MyDrive/data/fer2013.zip\n",
            "  inflating: fer2013.csv             \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sc5ZhrLuF6QU"
      },
      "source": [
        "!mv -n /content/fer2013.csv /content/FERPlus"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PkIOiSf1Guvh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "61870bdf-996a-4a82-b45e-a214547ac1c8"
      },
      "source": [
        "!mkdir data\n",
        "!python /content/FERPlus/src/generate_training_data.py -d /content/FERPlus/data -fer /content/FERPlus/fer2013.csv -ferplus /content/FERPlus/fer2013new.csv"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Start generating ferplus images.\n",
            "Done...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qXNXMNtgTNoO"
      },
      "source": [
        "import python file:\n",
        "\n",
        "```\n",
        "from google.colab import files\n",
        "src = list(files.upload().values())[0]\n",
        "open('mylib.py','wb').write(src)\n",
        "import mylib\n",
        "```\n",
        "\n",
        "\n",
        "```\n",
        "%load /content/FERPlus/src/models.py\n",
        "```\n",
        "\n",
        "\n",
        "```\n",
        "import sys\n",
        "sys.path.append('/content/FERPlus/src')\n",
        "\n",
        "\n",
        "import models\n",
        "```\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O7qolB-EyKX9"
      },
      "source": [
        "# Run the demo code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VXRtThfVIe7-"
      },
      "source": [
        "!python /content/FERPlus/src/train.py -d /content/FERPlus/data -m probability"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KpJKQGSXyRlZ"
      },
      "source": [
        "# Define the VGG13"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QNkAgqvZ0Pvy"
      },
      "source": [
        "from tensorflow import keras\n",
        "import tensorflow as tf\n",
        "from keras.layers import *\n",
        "from keras.models import Sequential\n",
        "\n",
        "def get_VGG13():\n",
        "  model = Sequential(name='VGG13')\n",
        "  model.add(InputLayer(input_shape=(64,64,1),name='input_1'))\n",
        "  model.add(Conv2D(64,(3,3),activation='relu',padding='same',name='block1_conv1'))\n",
        "  model.add(Conv2D(64,(3,3),activation='relu',padding='same',name='block1_conv2'))\n",
        "  model.add(MaxPooling2D((2,2),name='block1_pool'))\n",
        "  model.add(Dropout(0.25,name='block1_dropout'))\n",
        "\n",
        "  model.add(Conv2D(128,(3,3),activation='relu',padding='same',name='block2_conv1'))\n",
        "  model.add(Conv2D(128,(3,3),activation='relu',padding='same',name='block2_conv2'))\n",
        "  model.add(MaxPooling2D((2,2),name='block2_pool'))\n",
        "  model.add(Dropout(0.25,name='block2_dropout'))\n",
        "\n",
        "  model.add(Conv2D(256,(3,3),activation='relu',padding='same',name='block3_conv1'))\n",
        "  model.add(Conv2D(256,(3,3),activation='relu',padding='same',name='block3_conv2'))\n",
        "  model.add(Conv2D(256,(3,3),activation='relu',padding='same',name='block3_conv3'))\n",
        "  model.add(MaxPooling2D((2,2),name='block3_pool'))\n",
        "  model.add(Dropout(0.25,name='block3_dropout'))\n",
        "\n",
        "\n",
        "  model.add(Conv2D(256,(3,3),activation='relu',padding='same',name='block4_conv1'))\n",
        "  model.add(Conv2D(256,(3,3),activation='relu',padding='same',name='block4_conv2'))\n",
        "  model.add(Conv2D(256,(3,3),activation='relu',padding='same',name='block4_conv3'))\n",
        "  model.add(MaxPooling2D((2,2),name='block4_pool'))\n",
        "  model.add(Dropout(0.25,name='block4_dropout'))\n",
        "\n",
        "  model.add(Flatten(name='flatten'))\n",
        "  model.add(Dense(1024,activation='relu',name='fc1'))\n",
        "  model.add(Dropout(0.5,name='fc_dropout1'))\n",
        "  model.add(Dense(1024,activation='relu',name='fc2'))\n",
        "  model.add(Dropout(0.5,name='fc_dropout2'))\n",
        "  model.add(Dense(8,activation='softmax',name='predictions'))\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKCadatSh89L"
      },
      "source": [
        "# Data preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sAO8UARkGex7"
      },
      "source": [
        "import numpy as np\n",
        "import ossaudiodev\n",
        "import matplotlib.pyplot as plt\n",
        "from PIL import Image\n",
        "import sys\n",
        "import csv\n",
        "import os\n",
        "\n",
        "\n",
        "def show_img(img_arr):\n",
        "  img = img_arr.reshape((64,64))\n",
        "  img = Image.fromarray(img)\n",
        "  plt.imshow(img ,cmap='gray')\n",
        "  plt.show()\n",
        "  return None\n",
        "\n",
        "def get_image_name_list(dir_path):\n",
        "  return [os.path.join(dir_path,f) for f in os.listdir(dir_path) if f.endswith('.png')]\n",
        "\n",
        "def get_image_arr(dir_path, color_mode=\"grayscale\" ,target_size=(64,64)):\n",
        "  img_list = get_image_name_list(dir_path)\n",
        "  img_list.sort()\n",
        "  image_list = []\n",
        "  for fname in img_list:\n",
        "    image = keras.preprocessing.image.load_img(fname,color_mode=color_mode,target_size=target_size)\n",
        "    image_arr = keras.preprocessing.image.img_to_array(image)\n",
        "    image_list.append(image_arr)\n",
        "  input_arr = np.array(image_list)\n",
        "  return input_arr\n",
        "\n",
        "def get_label(dir_path, mode='crossentropy'):\n",
        "  fname = 'label.csv'\n",
        "  label_path = os.path.join(dir_path,fname)\n",
        "  with open(label_path) as csvfile:\n",
        "    label_file = csv.reader(csvfile) \n",
        "    label_list = []\n",
        "    for row in label_file:\n",
        "      emotion_raw = list(map(float, row[2:len(row)]))\n",
        "      emotion = process_data(emotion_raw=emotion_raw,mode=mode)\n",
        "      label_list.append(emotion)\n",
        "  return label_list\n",
        "\n",
        "def process_data(emotion_raw, mode='crossentropy'):\n",
        "    '''\n",
        "    Based on https://arxiv.org/abs/1608.01041, we process the data differently depend on the training mode:\n",
        "\n",
        "    Majority: return the emotion that has the majority vote, or unknown if the count is too little.\n",
        "    Probability or Crossentropty: convert the count into probability distribution.abs\n",
        "    Multi-target: treat all emotion with 30% or more votes as equal.\n",
        "    '''        \n",
        "    size = len(emotion_raw)\n",
        "    emotion_unknown     = [0.0] * size\n",
        "    emotion_unknown[-2] = 1.0\n",
        "\n",
        "    # remove emotions with a single vote (outlier removal) \n",
        "    for i in range(size):\n",
        "        if emotion_raw[i] < 1.0 + sys.float_info.epsilon:\n",
        "            emotion_raw[i] = 0.0\n",
        "\n",
        "    sum_list = sum(emotion_raw)\n",
        "    emotion = [0.0] * size \n",
        "\n",
        "    if mode == 'majority': \n",
        "        # find the peak value of the emo_raw list \n",
        "        maxval = max(emotion_raw) \n",
        "        if maxval > 0.5*sum_list: \n",
        "            emotion[np.argmax(emotion_raw)] = maxval \n",
        "        else: \n",
        "            emotion = emotion_unknown   # force setting as unknown \n",
        "    elif (mode == 'probability') or (mode == 'crossentropy'):\n",
        "        sum_part = 0\n",
        "        count = 0\n",
        "        valid_emotion = True\n",
        "        while sum_part < 0.75*sum_list and count < 3 and valid_emotion:\n",
        "            maxval = max(emotion_raw) \n",
        "            for i in range(size): \n",
        "                if emotion_raw[i] == maxval: \n",
        "                    emotion[i] = maxval\n",
        "                    emotion_raw[i] = 0\n",
        "                    sum_part += emotion[i]\n",
        "                    count += 1\n",
        "                    if i >= 8:  # unknown or non-face share same number of max votes \n",
        "                        valid_emotion = False\n",
        "                        if sum(emotion) > maxval:   # there have been other emotions ahead of unknown or non-face\n",
        "                            emotion[i] = 0\n",
        "                            count -= 1\n",
        "                        break\n",
        "        if sum(emotion) <= 0.5*sum_list or count > 3: # less than 50% of the votes are integrated, or there are too many emotions, we'd better discard this example\n",
        "            emotion = emotion_unknown   # force setting as unknown \n",
        "    elif mode == 'multi_target':\n",
        "        threshold = 0.3\n",
        "        for i in range(size): \n",
        "            if emotion_raw[i] >= threshold*sum_list: \n",
        "                emotion[i] = emotion_raw[i] \n",
        "        if sum(emotion) <= 0.5 * sum_list: # less than 50% of the votes are integrated, we discard this example \n",
        "            emotion = emotion_unknown   # set as unknown \n",
        "                            \n",
        "    return [float(i)/sum(emotion) for i in emotion]\n",
        "\n",
        "def get_data(dir_path,mode='crossentropy'):  # not unknown or non-face \n",
        "  imgs = get_image_arr(dir_path)\n",
        "  labels = get_label(dir_path,mode)\n",
        "  if len(imgs)==len(labels):\n",
        "    data_X = []\n",
        "    data_y = []\n",
        "    for i in range(len(labels)): \n",
        "      emotion = labels[i]\n",
        "      idx = np.argmax(labels[i])\n",
        "      if idx < 8: \n",
        "          emotion = emotion[:-2]\n",
        "          emotion = [float(j)/sum(emotion) for j in emotion]\n",
        "          data_X.append(imgs[i])\n",
        "          data_y.append(emotion)\n",
        "    data_X = np.array(data_X)\n",
        "    data_y = np.array(data_y)\n",
        "    return data_X,data_y\n",
        "  else:\n",
        "    print(\"Inconsistent number of images and labels\")\n",
        "    return None,None"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B3rSwjq2BUQH"
      },
      "source": [
        "import os \n",
        "\n",
        "base_dir = '/content/FERPlus/data'\n",
        "train_dir = os.path.join(base_dir,'FER2013Train')\n",
        "validation_dir = os.path.join(base_dir,'FER2013Valid')\n",
        "test_dir = os.path.join(base_dir,'FER2013Test')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YNfbhM_rgguc"
      },
      "source": [
        "train_X,train_y = get_data(train_dir)\n",
        "validation_X,validation_y = get_data(validation_dir)\n",
        "test_X,test_y = get_data(test_dir)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xeLeC2yHifxy"
      },
      "source": [
        "# Fit the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_mjsIl8ijUa"
      },
      "source": [
        "model = get_VGG13()\n",
        "model.compile(optimizer=keras.optimizers.RMSprop(),loss= 'kullback_leibler_divergence', metrics=['kullback_leibler_divergence'])\n",
        "history = model.fit(train_X, train_y, batch_size=256, epochs=10, validation_data=(validation_X,validation_y))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        },
        "id": "A-oz1H4ekomh",
        "outputId": "e9d79366-a20d-4d73-8900-8fec3c8b3ecd"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(1,len(loss)+1)\n",
        "\n",
        "plt.plot(epochs,loss,'ro',label='Training loss')\n",
        "plt.plot(epochs,val_loss,'r',label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-208-cc89ae702823>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mval_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'history' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vUdSteR_AN0X"
      },
      "source": [
        "# create needed folders.\n",
        "import os,shutil\n",
        "output_model_path   = os.path.join(base_folder, R'models')\n",
        "output_model_folder = os.path.join(output_model_path, model_name + '_' + training_mode)\n",
        "if not os.path.exists(output_model_folder):\n",
        "    os.makedirs(output_model_folder)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}