Text Classification - FYP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification Engine for Sensor Fusion\n",
    "\n",
    "First, let us load and preprocess text transcriptions.\n",
    "\n",
    "**Note**: Running this python script require `nltk` libraries to be set up in prior. Therefore, please download the following dependencies by uncommenting the line `nltk.download()`.\n",
    "\n",
    "1. `perluniprops`\n",
    "2. `punkt`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "# from sklearn.externals import joblib\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.tokenize import word_tokenize \n",
    "from sklearn.utils import shuffle\n",
    "from nltk.tokenize.treebank import TreebankWordDetokenizer as Detok\n",
    "from nltk.tokenize.treebank import TreebankWordDetokenizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from tensorflow.keras.layers import *\n",
    "import matplotlib.pyplot as plt\n",
    "# from tensorflow.keras.utils.np_utils import to_categorical\n",
    "from tensorflow.keras.regularizers import l2\n",
    "# from tensorflow.keras.utils.vis_utils import plot_model\n",
    "# from yellowbrick.classifier import ClassificationReport\n",
    "from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n",
    "from tensorflow.keras.models import load_model\n",
    "import os.path\n",
    "from imblearn.over_sampling import ADASYN, SMOTE, RandomOverSampler\n",
    "from imblearn.pipeline import make_pipeline\n",
    "from imblearn.keras import BalancedBatchGenerator\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.utils.multiclass import unique_labels\n",
    "from sklearn.utils import class_weight\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import the dataset and shuffle it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                   sentence  operation\n",
      "500       I want the details of this bottle          2\n",
      "601             Can you describe this chair          2\n",
      "104                    Point out the bottle          1\n",
      "400        What are the features of the pen          2\n",
      "290                Can you help find my pen          1\n",
      "..                                      ...        ...\n",
      "416  What are the properties of the monitor          2\n",
      "219             Find where the keyboard are          1\n",
      "162              Show me where the mouse is          1\n",
      "285              I need to find my keyboard          1\n",
      "621                        look at the book          3\n",
      "\n",
      "[709 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "filepath = 'data/fyp_dataset.txt'\n",
    "df = pd.read_csv(filepath, names=['sentence', 'operation'], sep=', ', engine='python')\n",
    "df = shuffle(df)\n",
    "sentences = df['sentence'].values\n",
    "y = df['operation'].values\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords_set = [\"i\", \"me\", \"my\", \"myself\", \"we\", \"our\", \"ours\", \"ourselves\", \"you\", \"your\", \"yours\", \"yourself\",\n",
    "                  \"yourselves\", \"he\", \"him\", \"his\", \"himself\", \"she\", \"her\", \"hers\", \"herself\", \"it\", \"its\",\n",
    "                  \"itself\", \"they\", \"them\", \"their\", \"theirs\", \"themselves\", \"which\", \"who\", \"whom\", \"these\",\n",
    "                  \"those\", \"am\", \"is\", \"are\", \"was\", \"were\", \"be\", \"been\", \"being\", \"have\", \"has\", \"had\", \"having\",\n",
    "                  \"do\", \"does\", \"did\", \"doing\", \"a\", \"an\", \"the\", \"and\", \"but\", \"if\", \"or\", \"because\", \"as\",\n",
    "                  \"until\", \"while\", \"of\", \"at\", \"by\", \"for\", \"with\", \"against\", \"into\", \"through\", \"during\",\n",
    "                  \"before\", \"after\", \"above\", \"below\", \"to\", \"from\", \"up\", \"down\", \"in\", \"out\", \"on\", \"off\", \"over\",\n",
    "                  \"under\", \"again\", \"further\", \"then\", \"once\", \"here\", \"there\", \"when\", \"why\", \"how\", \"all\", \"any\",\n",
    "                  \"both\", \"each\", \"few\", \"more\", \"most\", \"other\", \"some\", \"such\", \"no\", \"nor\", \"not\", \"only\", \"own\",\n",
    "                  \"same\", \"so\", \"than\", \"too\", \"very\", \"s\", \"t\", \"don\", \"should\", \"now\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before removing the stopwords, we need to tokenize the sentences. Afterwards, we remove the stopwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_stop_words(sentences):\n",
    "    filtered_sentences = []\n",
    "    detokenizer = Detok()\n",
    "    for sentence in sentences:\n",
    "        tokenized_sentence = word_tokenize(sentence)\n",
    "        filtered_sentence = [word for word in tokenized_sentence if word not in stopwords_set]\n",
    "        filtered_sentence = [] \n",
    "        for w in tokenized_sentence: \n",
    "            if w not in stopwords_set: \n",
    "                filtered_sentence.append(w)\n",
    "        filtered_sentences.append(filtered_sentence)\n",
    "    return filtered_sentences\n",
    "        \n",
    "def detokenize(filtered_sentences):\n",
    "    detokenized_sentences = []\n",
    "    for sentence in filtered_sentences:\n",
    "        detokenized_sentences.append(TreebankWordDetokenizer().detokenize(sentence))\n",
    "    return detokenized_sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us detokenize the output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_sentences = filter_stop_words(sentences)\n",
    "detokenized_sentences = detokenize(filtered_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['locate phones']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let us assign the detokenized sentences back to the `pandas` dataframe. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>operation</th>\n",
       "      <th>filtered_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>Give an account of this hand</td>\n",
       "      <td>2</td>\n",
       "      <td>Give account this hand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>Give me the specification of the chair</td>\n",
       "      <td>2</td>\n",
       "      <td>Give specification chair</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>Detect cup</td>\n",
       "      <td>1</td>\n",
       "      <td>Detect cup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>What are the features of the monitor</td>\n",
       "      <td>2</td>\n",
       "      <td>What features monitor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>What are the locations of cups</td>\n",
       "      <td>1</td>\n",
       "      <td>What locations cups</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   sentence  operation  \\\n",
       "435            Give an account of this hand          2   \n",
       "590  Give me the specification of the chair          2   \n",
       "112                              Detect cup          1   \n",
       "405    What are the features of the monitor          2   \n",
       "79           What are the locations of cups          1   \n",
       "\n",
       "            filtered_sentence  \n",
       "435    Give account this hand  \n",
       "590  Give specification chair  \n",
       "112                Detect cup  \n",
       "405     What features monitor  \n",
       "79        What locations cups  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['filtered_sentence'] = detokenized_sentences\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the model.\n",
    "\n",
    "Here, we will test out several models and test the accuracy parameters, in order to arrive at a final model. We will also use a grid-search methodology for obtaining the best hyperparameters for the chosen model.\n",
    "\n",
    "However, before this step, we need to understand the distribution of the dataset. For this, we will use `matplotlib` to plot the dataset w.r.t the labels. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = ['1', '2', '3']\n",
    "\n",
    "x = df['filtered_sentence'].values\n",
    "y = df['operation']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.9438202247191011\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.92      1.00      0.96        69\n",
      "           2       0.95      1.00      0.98        82\n",
      "           3       1.00      0.63      0.77        27\n",
      "\n",
      "    accuracy                           0.94       178\n",
      "   macro avg       0.96      0.88      0.90       178\n",
      "weighted avg       0.95      0.94      0.94       178\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sgd = Pipeline([\n",
    "    ('vect', CountVectorizer()),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', SGDClassifier(loss='hinge', penalty='l2',alpha=1e-3, random_state=42, max_iter=5, tol=None))\n",
    "])\n",
    "\n",
    "sgd.fit(X_train, y_train)\n",
    "\n",
    "y_pred = sgd.predict(X_test)\n",
    "\n",
    "print('Accuracy %s' % accuracy_score(y_pred, y_test))\n",
    "\n",
    "print(classification_report(y_test, y_pred,target_names=tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pickle import load, dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sgd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-180fddf8804c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msgd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"model.pkl\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"wb\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'sgd' is not defined"
     ]
    }
   ],
   "source": [
    "dump(sgd, open(\"model.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load(open(\"model.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_test' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-5e72cafd92f5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Accuracy %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassification_report\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtarget_names\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X_test' is not defined"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "\n",
    "print('Accuracy %s' % accuracy_score(y_pred, y_test))\n",
    "\n",
    "print(classification_report(y_test, y_pred,target_names=tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_dictionary = {\n",
    "    'laptops': {'operation': None, 'object_id': 6, 'multiple': True, 'pointing': False},\n",
    "    'phones': {'operation': None, 'object_id': 4, 'multiple': True, 'pointing': False},\n",
    "    'books': {'operation': None, 'object_id': 0, 'multiple': True, 'pointing': False},\n",
    "    'bottles': {'operation': None, 'object_id': 3, 'multiple': True, 'pointing': False},\n",
    "    'pens': {'operation': None, 'object_id': 7, 'multiple': True, 'pointing': False},\n",
    "    'cups': {'operation': None, 'object_id': 9, 'multiple': True, 'pointing': False},\n",
    "    'keyboards': {'operation': None, 'object_id': 8, 'multiple': True, 'pointing': False},\n",
    "    'mouses': {'operation': None, 'object_id': 5, 'multiple': True, 'pointing': False},\n",
    "    'monitors': {'operation': None, 'object_id': 2, 'multiple': True, 'pointing': False},\n",
    "    'laptop': {'operation': None, 'object_id': 6, 'multiple': False, 'pointing': False},\n",
    "    'phone': {'operation': None, 'object_id': 4, 'multiple': False, 'pointing': False},\n",
    "    'book': {'operation': None, 'object_id': 0, 'multiple': False, 'pointing': False},\n",
    "    'bottle': {'operation': None, 'object_id': 3, 'multiple': False, 'pointing': False},\n",
    "    'pen': {'operation': None, 'object_id': 7, 'multiple': False, 'pointing': False},\n",
    "    'cup': {'operation': None, 'object_id': 9, 'multiple': False, 'pointing': False},\n",
    "    'keyboard': {'operation': None, 'object_id': 8, 'multiple': False, 'pointing': False},\n",
    "    'mouse': {'operation': None, 'object_id': 5, 'multiple': False, 'pointing': False},\n",
    "    'monitor': {'operation': None, 'object_id': 2, 'multiple': False, 'pointing': False}\n",
    "}\n",
    "labels = ['Locate', 'Describe', 'Invalid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"locate the phones\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Locate\n"
     ]
    }
   ],
   "source": [
    "pred = model.predict([text])\n",
    "print(labels[pred[0]-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'operation': None, 'object_id': 4, 'multiple': True, 'pointing': False}\n"
     ]
    }
   ],
   "source": [
    "_pointing = False\n",
    "for token in tokens[0]:\n",
    "    if token in [\"this\", \"that\"]:\n",
    "        _pointing = True\n",
    "    elif token in name_dictionary:\n",
    "        command = name_dictionary[token]\n",
    "        command[\"pointing\"] = _pointing\n",
    "        print(command)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = filter_stop_words([text])\n",
    "filtered_commands = detokenize(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}