Spam_class_NLP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spam detection using NLP\n",
    "## Yeison Lopez"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This project is an example of spam classification using NLP. Only for educational purposes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nltk.download_shell()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using the UCI Datasets\n",
    "messages = [line.rstrip() for line in open('smsspamcollection/SMSSpamcollection')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5574\n"
     ]
    }
   ],
   "source": [
    "print(len(messages))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ham\\tWhat you thinked about me. First time you saw me in class.'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages[50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ham\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
      "\n",
      "\n",
      "1 ham\tOk lar... Joking wif u oni...\n",
      "\n",
      "\n",
      "2 spam\tFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's\n",
      "\n",
      "\n",
      "3 ham\tU dun say so early hor... U c already then say...\n",
      "\n",
      "\n",
      "4 ham\tNah I don't think he goes to usf, he lives around here though\n",
      "\n",
      "\n",
      "5 spam\tFreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, Â£1.50 to rcv\n",
      "\n",
      "\n",
      "6 ham\tEven my brother is not like to speak with me. They treat me like aids patent.\n",
      "\n",
      "\n",
      "7 ham\tAs per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune\n",
      "\n",
      "\n",
      "8 spam\tWINNER!! As a valued network customer you have been selected to receivea Â£900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.\n",
      "\n",
      "\n",
      "9 spam\tHad your mobile 11 months or more? U R entitled to Update to the latest colour mobiles with camera for Free! Call The Mobile Update Co FREE on 08002986030\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print first 10 messages\n",
    "for mess_no,message in enumerate(messages[:10]):\n",
    "    print(mess_no,message)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the csv and understand the \\t separator and the column names\n",
    "messages = pd.read_csv('smsspamcollection/SMSSpamcollection',sep='\\t',\n",
    "                      names=['label','message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exploratory data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5572</td>\n",
       "      <td>5572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>2</td>\n",
       "      <td>5169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>ham</td>\n",
       "      <td>Sorry, I'll call later</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>4825</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       label                 message\n",
       "count   5572                    5572\n",
       "unique     2                    5169\n",
       "top      ham  Sorry, I'll call later\n",
       "freq    4825                      30"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Quick stats\n",
    "messages.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">message</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>unique</th>\n",
       "      <th>top</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ham</th>\n",
       "      <td>4825</td>\n",
       "      <td>4516</td>\n",
       "      <td>Sorry, I'll call later</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spam</th>\n",
       "      <td>747</td>\n",
       "      <td>653</td>\n",
       "      <td>Please call our customer service representativ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      message                                                               \n",
       "        count unique                                                top freq\n",
       "label                                                                       \n",
       "ham      4825   4516                             Sorry, I'll call later   30\n",
       "spam      747    653  Please call our customer service representativ...    4"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# High level view of the data\n",
    "messages.groupby('label').describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have 4825 ham vs 747 spam with their most popular texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a new column to detect how long the messages are\n",
    "messages['length'] = messages['message'].apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message  length\n",
       "0   ham  Go until jurong point, crazy.. Available only ...     111\n",
       "1   ham                      Ok lar... Joking wif u oni...      29\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...     155\n",
       "3   ham  U dun say so early hor... U c already then say...      49\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...      61"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualize the length of the messages\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x17aff5873a0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASM0lEQVR4nO3da7BdZX3H8e+PYEHUjjAkmCbBEzsZFRwRjNSWXhS0RFGCnWLjVCfjoGlrrNo6o4nTqfZFZvKiRem0tKZoG68Y8UIqrYqx6HRGCUFpMVyGjESISUmkWqjjhCb++2KvrGySc5J9kqyzT87+fmbO7LWevdbe//3AOb886/LsVBWSJAGcMuwCJEnTh6EgSWoZCpKklqEgSWoZCpKk1qnDLuB4nH322TU2NjbsMiTppHLnnXf+qKpmj/fcSR0KY2NjbNmyZdhlSNJJJckPJnrOw0eSpJahIElqGQqSpJahIElqGQqSpJahIElqGQqSpFanoZBke5K7k9yVZEvTdlaSW5M80Dye2bf96iTbktyf5PIua5MkHW4qRgovr6oXVdXiZn0VsKmqFgGbmnWSnAcsA84HlgDXJ5k1BfVJkhrDOHy0FFjfLK8Hruprv7Gq9lbVg8A24OIh1MfYqlsYW3XLMN5akoaq61Ao4KtJ7kyyomk7p6p2ATSPc5r2ecDDffvuaNqeJMmKJFuSbNmzZ0+HpUvS6Ol67qNLqmpnkjnArUnuO8K2GaftsO8Krap1wDqAxYsXd/pdogdGC9vXXtHl20jStNHpSKGqdjaPu4Ev0Dsc9EiSuQDN4+5m8x3Agr7d5wM7u6xPkvRknYVCkqclecaBZeC3ge8BG4HlzWbLgZub5Y3AsiSnJVkILAI2d1WfJOlwXR4+Ogf4QpID7/OpqvpykjuADUmuAR4Crgaoqq1JNgD3APuAlVW1v8P6JEmH6CwUqur7wAXjtD8KXDbBPmuANV3VJEk6Mu9oliS1DAVJUstQkCS1DAVJUstQkCS1DAVJUstQkCS1DAVJUstQkCS1DAVJUstQkCS1DAVJUstQGIBfzylpVBgKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqRW56GQZFaS7yb5UrN+VpJbkzzQPJ7Zt+3qJNuS3J/k8q5rkyQ92VSMFN4J3Nu3vgrYVFWLgE3NOknOA5YB5wNLgOuTzJqC+iRJjU5DIcl84Arghr7mpcD6Znk9cFVf+41VtbeqHgS2ARd3WZ8k6cm6Hil8CHgP8PO+tnOqahdA8zinaZ8HPNy33Y6m7UmSrEiyJcmWPXv2dFO1JI2ozkIhyWuA3VV156C7jNNWhzVUrauqxVW1ePbs2cdVoyTpyU7t8LUvAa5M8mrgdOAXk3wCeCTJ3KralWQusLvZfgewoG//+cDODuuTJB2is5FCVa2uqvlVNUbvBPLXq+qNwEZgebPZcuDmZnkjsCzJaUkWAouAzV3VJ0k6XJcjhYmsBTYkuQZ4CLgaoKq2JtkA3APsA1ZW1f4h1CdJI2tKQqGqbgNua5YfBS6bYLs1wJqpqEmSdDjvaJYktQwFSVLLUJAktQwFSVLLUJAktQwFSVLLUJAktQwFSVLLUJiEsVW3MLbqlmGXIUmdMRQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1D4Rh4v4KkmcpQkCS1DIUZwJGLpBPFUJAktQwFSVLLUJAktQwFSVLLUJAktQwFSVLLUJAktQwFSVJroFBI8oKuC5EkDd+gI4W/T7I5yduSPLPTiiRJQzNQKFTVrwO/DywAtiT5VJJXdlqZJGnKDXxOoaoeAP4MeC/wW8BfJ7kvye90VZwkaWoNek7hhUk+CNwLXAq8tqqe3yx/sMP6JElTaNCRwt8A3wEuqKqVVfUdgKraSW/0cJgkpzfnIf4jydYkf9G0n5Xk1iQPNI9n9u2zOsm2JPcnufz4PpokabIGDYVXA5+qqp8BJDklyRkAVfXxCfbZC1xaVRcALwKWJHkpsArYVFWLgE3NOknOA5YB5wNLgOuTzDq2jyVJOhaDhsLXgKf2rZ/RtE2oev63WX1K81PAUmB9074euKpZXgrcWFV7q+pBYBtw8YD1SZJOgEFD4fS+P/A0y2ccbacks5LcBewGbq2q24FzqmpX8zq7gDnN5vOAh/t239G0HfqaK5JsSbJlz549A5YvSRrEoKHw0yQXHVhJ8mLgZ0fbqar2V9WLgPnAxUe5CS7jvcQ4r7muqhZX1eLZs2cPULokaVCnDrjdu4DPJtnZrM8Ffm/QN6mqnyS5jd65gkeSzK2qXUnm0htFQG9ksKBvt/nATiRJU2bQm9fuAJ4H/BHwNuD5VXXnkfZJMvvA3c9Jngq8ArgP2AgsbzZbDtzcLG8EliU5LclCYBGweXIfR5J0PAYdKQC8BBhr9rkwCVX1sSNsPxdY31xBdAqwoaq+lORbwIYk1wAPAVcDVNXWJBuAe4B9wMqq2j/pTyRJOmYDhUKSjwO/DNwFHPhDXcCEoVBV/wlcOE77o8BlE+yzBlgzSE2SpBNv0JHCYuC8qjrsxO8oG1t1CwDb114x5Eok6cQY9Oqj7wHP6rIQSdLwDTpSOBu4J8lmencqA1BVV3ZSlSRpKAYNhQ90WYQkaXoYKBSq6htJng0sqqqvNfMeOS+RJM0wg06d/VbgJuDDTdM84ItdFSVJGo5BTzSvBC4BHoP2C3fmHHEPSdJJZ9BQ2FtVTxxYSXIq48xLJEk6uQ0aCt9I8j7gqc13M38W+OfuypIkDcOgobAK2APcDfwB8C9M8I1rkqST16BXH/0c+IfmR5I0Qw0699GDjP/dBs854RVJkoZmMnMfHXA6vZlNzzrx5UiShmnQ71N4tO/nh1X1IeDSjmuTJE2xQQ8fXdS3egq9kcMzOqlIkjQ0gx4++qu+5X3AduD1J7waSdJQDXr10cu7LkSSNHyDHj760yM9X1XXnphyJEnDNJmrj14CbGzWXwt8E3i4i6IkScMxmS/ZuaiqHgdI8gHgs1X1lq4KkyRNvUGnuTgXeKJv/Qlg7IRXI0kaqkFHCh8HNif5Ar07m18HfKyzqiRJQzHo1Udrkvwr8BtN05ur6rvdlSVJGoZBDx8BnAE8VlXXATuSLOyoJknSkAz6dZzvB94LrG6angJ8oquiJEnDMehI4XXAlcBPAapqJ05zIUkzzqCh8ERVFc302Ume1l1JkqRhGTQUNiT5MPDMJG8FvoZfuCNJM85Rrz5KEuAzwPOAx4DnAn9eVbd2XJskaYodNRSqqpJ8sapeDBgEkjSDDXr46NtJXtJpJZKkoRv0juaXA3+YZDu9K5BCbxDxwq4KkyRNvSOGQpJzq+oh4FWTfeEkC+hNhfEs4OfAuqq6LslZ9M5RjNF8WU9V/bjZZzVwDbAfeEdVfWWy7zsdjK26BYDta68YciWSNDlHO3z0RYCq+gFwbVX9oP/nKPvuA95dVc8HXgqsTHIesArYVFWLgE3NOs1zy4DzgSXA9UlmHesHkyRN3tEOH6Vv+TmTeeGq2gXsapYfT3IvMA9YCrys2Ww9cBu9u6WXAjdW1V7gwSTbgIuBb03mfYfpwAhBkk5WRwuFmmB5UpKMARcCtwPnNIFBVe1KMqfZbB7w7b7ddjRth77WCmAFwLnnnnusJZ1QhoGkmeJoh48uSPJYkseBFzbLjyV5PMljg7xBkqcDnwPeVVVH2ifjtB0WRFW1rqoWV9Xi2bNnD1KCJGlARxwpVNVxHdNP8hR6gfDJqvp80/xIkrnNKGEusLtp3wEs6Nt9PrDzeN5/pnOEIulEm8zU2ZPS3An9EeDeqrq276mNwPJmeTlwc1/7siSnNdNyLwI2d1WfJOlwg96ncCwuAd4E3J3krqbtfcBaenMpXQM8BFwNUFVbk2wA7qF35dLKqtrfYX2SpEN0FgpV9e+Mf54A4LIJ9lkDrOmqJknSkXV2+EiSdPIxFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktQyFCRJLUNhCoytusUZTSWdFAwFSVKry1lS1RFHHZK6Yih0yD/ekk42Hj6SJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CYQs6WKmm6c+6jk4iBIqlrhsJJwDCQNFU8fCRJahkKM4jnLCQdL0NBktQyFCRJLUNBktTq7OqjJB8FXgPsrqoXNG1nAZ8BxoDtwOur6sfNc6uBa4D9wDuq6itd1TZdHXo+YPvaK4ZUiaRR1eVI4Z+AJYe0rQI2VdUiYFOzTpLzgGXA+c0+1yeZ1WFtkqRxdBYKVfVN4L8PaV4KrG+W1wNX9bXfWFV7q+pBYBtwcVe1SZLGN9XnFM6pql0AzeOcpn0e8HDfdjuathnJS0clTVfT5Y7mjNNW426YrABWAJx77rld1tS5owWDwSFpqk31SOGRJHMBmsfdTfsOYEHfdvOBneO9QFWtq6rFVbV49uzZnRYrSaNmqkNhI7C8WV4O3NzXvizJaUkWAouAzVNcmySNvC4vSf008DLg7CQ7gPcDa4ENSa4BHgKuBqiqrUk2APcA+4CVVbW/q9okSePrLBSq6g0TPHXZBNuvAdZ0VY8k6ei8o1mS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEmt6TLNxbTgtBKSRp0jBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSa6RDYWzVLd7FLEl9nOYCp7eQpANGeqQgSXoyQ0GS1DIUJEktQ0GS1DIUJEktQ2EG8lJbScfKUJAktQwFSVLLUJAktQyFEeA5BkmDcpqLGcwgkDRZjhQkSS1DQZLUMhQkSS1DYQSNd+LZk9GSYBqeaE6yBLgOmAXcUFVrh1zSjDFeEEx2nwO2r73ihNQkaXqZVqGQZBbwt8ArgR3AHUk2VtU9w61sdBwIgaP90T80LCa7/aFO9PsNw6B9J01n0yoUgIuBbVX1fYAkNwJLAUNhik32UNLxHno61vc78Ad4ovWjmegP+Hj7T/Reg9Y42eenyvHUOV0+w4k2nT9X17Wlqjp54WOR5HeBJVX1lmb9TcCvVNXb+7ZZAaxoVp8L3H+Mb3c28KPjKHcmsS8Osi8Osi8Omml98eyqmj3eE9NtpJBx2p6UWlW1Dlh33G+UbKmqxcf7OjOBfXGQfXGQfXHQKPXFdLv6aAewoG99PrBzSLVI0siZbqFwB7AoycIkvwAsAzYOuSZJGhnT6vBRVe1L8nbgK/QuSf1oVW3t6O2O+xDUDGJfHGRfHGRfHDQyfTGtTjRLkoZruh0+kiQNkaEgSWqNXCgkWZLk/iTbkqwadj1dS7Igyb8luTfJ1iTvbNrPSnJrkgeaxzP79lnd9M/9SS4fXvXdSDIryXeTfKlZH8m+SPLMJDclua/5/+NXR7gv/qT5/fhekk8nOX1U+2KkQqFvGo1XAecBb0hy3nCr6tw+4N1V9XzgpcDK5jOvAjZV1SJgU7NO89wy4HxgCXB9028zyTuBe/vWR7UvrgO+XFXPAy6g1ycj1xdJ5gHvABZX1QvoXeSyjBHsCxixUKBvGo2qegI4MI3GjFVVu6rqO83y4/R+8efR+9zrm83WA1c1y0uBG6tqb1U9CGyj128zQpL5wBXADX3NI9cXSX4R+E3gIwBV9URV/YQR7IvGqcBTk5wKnEHv/qiR7ItRC4V5wMN96zuatpGQZAy4ELgdOKeqdkEvOIA5zWYzvY8+BLwH+Hlf2yj2xXOAPcA/NofSbkjyNEawL6rqh8BfAg8Bu4D/qaqvMoJ9AaMXCkedRmOmSvJ04HPAu6rqsSNtOk7bjOijJK8BdlfVnYPuMk7bjOgLev8yvgj4u6q6EPgpzeGRCczYvmjOFSwFFgK/BDwtyRuPtMs4bTOiL2D0QmEkp9FI8hR6gfDJqvp80/xIkrnN83OB3U37TO6jS4Ark2ynd+jw0iSfYDT7Ygewo6pub9ZvohcSo9gXrwAerKo9VfV/wOeBX2M0+2LkQmHkptFIEnrHje+tqmv7ntoILG+WlwM397UvS3JakoXAImDzVNXbpapaXVXzq2qM3n/7r1fVGxnNvvgv4OEkz22aLqM3Rf3I9QW9w0YvTXJG8/tyGb1zb6PYF9NrmouuTfE0GtPFJcCbgLuT3NW0vQ9YC2xIcg29X4qrAapqa5IN9P5A7ANWVtX+qS97So1qX/wx8MnmH0jfB95M7x+KI9UXVXV7kpuA79D7bN+lN63F0xmxvgCnuZAk9Rm1w0eSpCMwFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktT6fzsSw97ZrokEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "messages['length'].plot.hist(bins=150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    5572.000000\n",
       "mean       80.489950\n",
       "std        59.942907\n",
       "min         2.000000\n",
       "25%        36.000000\n",
       "50%        62.000000\n",
       "75%       122.000000\n",
       "max       910.000000\n",
       "Name: length, dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages['length'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"For me the love should start with attraction.i should feel that I need her every time around me.she should be the first thing which comes in my thoughts.I would start the day and end it with her.she should be there every time I dream.love will be then when my every breath has her name.my life should happen around her.my life will be named to her.I would cry for her.will give all my happiness and take all her sorrows.I will be ready to fight with anyone for her.I will be in love when I will be doing the craziest things for her.love will be when I don't have to proove anyone that my girl is the most beautiful lady on the whole planet.I will always be singing praises for her.love will be when I start up making chicken curry and end up makiing sambar.life will be the most beautiful then.will get every morning and thank god for the day because she is with me.I would like to say a lot..will tell later..\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find the longest message\n",
    "messages[messages['length'] == 910]['message'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([<matplotlib.axes._subplots.AxesSubplot object at 0x0000017AFF7EA760>,\n",
       "       <matplotlib.axes._subplots.AxesSubplot object at 0x0000017AFF714DC0>],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuIAAAEQCAYAAAD1URGwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAdGElEQVR4nO3df7RdZ13n8feHBAotAk17W9umJWHMFFuUH14Lwqgdg7YzZZHKmmIYwYBlMj+qqOMaSdW1qrMmM2HGX3U5dU1sqXEEQqg/GkWBGheyVKAEqNgk1AYa2rRpcpG24o+pJP3OH2dXTi83bXLvPee555z3a62us8+z9777u09v7v7c5z57P6kqJEmSJA3X01oXIEmSJE0ig7gkSZLUgEFckiRJasAgLkmSJDVgEJckSZIaMIhLkiRJDRjENRaSHEjy6tZ1SJIknSiDuCRJktSAQVySJElqwCCucfKSJJ9J8kiS9yZ5ZpLTk/x+kpkkD3XLKx/fIcmHk/y3JH+e5G+T/F6SM5K8K8nfJPlEklXtTkmSdDKSvD3J/Um+nOSuJGuT/EySW7prw5eTfCrJi/v22ZTkc926vUm+t2/dm5P8WZJfTPJwks8neWXXfl+SI0k2tDlbjTqDuMbJ64HLgdXANwNvpvc9fjPwfOAC4B+AX5m133rgTcB5wD8DPtrtswLYB1w3+NIlSQuV5ELgh4BvraqvAy4DDnSr1wHvo/ez/d3A7yZ5erfuc8C3A88Ffhb4zSTn9H3plwOfAc7o9t0OfCvwDcAbgV9J8uzBnZnGlUFc4+SXq+qBqvoS8HvAS6rqr6vqt6rq76vqy8Bm4Dtn7XdzVX2uqh4B/hD4XFX9UVUdpfdD+6VDPQtJ0nwdA04BLkry9Ko6UFWf69Z9sqpuqaqvAL8APBN4BUBVva+7fjxWVe8F7gYu6fu691TVzVV1DHgvcD7wX6vq0ar6EPCP9EK5dFIM4honD/Yt/z3w7CSnJvk/Sb6Q5G+AjwDPS7Ksb9vDfcv/MMd7ezkkaQRU1X7gR4GfAY4k2Z7k3G71fX3bPQYcBM4FSPIDSe7ohp48DLwIOLPvS8++LlBVXiu0YAZxjbsfBy4EXl5VzwG+o2tPu5IkSYNSVe+uqn9Bb0hiAe/oVp3/+DZJngasBB5I8nzg1+gNaTmjqp4H3InXCQ2BQVzj7uvo9VQ8nGQFjveWpLGV5MIk35XkFOD/0fv5f6xb/S1JXpdkOb1e80eBjwGn0QvsM93XeAu9HnFp4AziGne/BDwL+CK9H7gfaFuOJGmATgG20PuZ/yBwFvCT3bpbge8DHqJ3g/7rquorVbUX+Hl6N+ofBr4J+LMh160JlapqXYMkSdLAJPkZ4Buq6o2ta5H62SMuSZIkNWAQlyRJkhpwaIokSZLUgD3ikqRFk+Sd3ZTfd/a1/a8kn03ymSS/k+R5feuuTbK/m4r8sjZVS1IbBnFJ0mL6deDyWW23AS+qqm8G/gq4FiDJRcB64OJunxtmTbYlSWNteesCnsqZZ55Zq1atal2GJD2lT37yk1+sqqnWdbRUVR9JsmpW24f63n4M+Dfd8jpge1U9CtyTZD+9acU/+mTH8LogaZQ82bVhyQfxVatWsXv37tZlSNJTSvKF1jWMgB8E3tstn0cvmD/uYNf2NZJsBDYCXHDBBV4XJI2MJ7s2ODRFkjQUSX4KOAq86/GmOTab8wkCVbW1qqaranpqaqL/6CBpjCz5HnFJ0uhLsgF4DbC2vvq4roPA+X2brQQeGHZtktSKPeKSpIFKcjnwduC1VfX3fat2AuuTnJJkNbAGuL1FjZLUgj3ikqRFk+Q9wKXAmUkOAtfRe0rKKcBtSQA+VlX/oar2JNkB7KU3ZOWaqjrWpnJJGj6DuCRp0VTVG+ZovulJtt8MbB5cRZK0dDk0RZIkSWrAIC5JkiQ1YBCXJEmSGpiYMeKrNr3/Ce8PbLmiUSWSJEmLY3a+ATPOKLFHXJIkSWrAIC5JkiQ1YBCXJEmSGnjKIJ7knUmOJLmzr21FktuS3N29nt637tok+5PcleSyvvZvSfKX3bpfTjergyRJkjSJTqRH/NeBy2e1bQJ2VdUaYFf3niQXAeuBi7t9bkiyrNvnV4GN9KYwXjPH15QkSZImxlMG8ar6CPClWc3rgG3d8jbgyr727VX1aFXdA+wHLklyDvCcqvpoVRXwG337SJIkSRNnvo8vPLuqDgFU1aEkZ3Xt5wEf69vuYNf2lW55dnszPu5HkiRJLS32zZpzjfuuJ2mf+4skG5PsTrJ7ZmZm0YqTJEmSlor5BvHD3XATutcjXftB4Py+7VYCD3TtK+don1NVba2q6aqanpqammeJkiRJ0tI13yC+E9jQLW8Abu1rX5/klCSr6d2UeXs3jOXLSV7RPS3lB/r2kSRJkibOU44RT/Ie4FLgzCQHgeuALcCOJFcD9wJXAVTVniQ7gL3AUeCaqjrWfan/SO8JLM8C/rD7T5IkSZpITxnEq+oNx1m19jjbbwY2z9G+G3jRSVUnSZIkjSln1pQkSZIamO/jCyVJkrQEzX5Es49nXrrsEZckSZIaMIhLkiRJDRjEJUmSpAYM4pIkSVIDBnFJkiSpAYO4JEmS1IBBXJIkSWrAIC5JkiQ1YBCXJC2aJO9MciTJnX1tK5LcluTu7vX0vnXXJtmf5K4kl7WpWpLaMIhLkhbTrwOXz2rbBOyqqjXAru49SS4C1gMXd/vckGTZ8EqVpLYM4pKkRVNVHwG+NKt5HbCtW94GXNnXvr2qHq2qe4D9wCVDKVSSlgCDuCRp0M6uqkMA3etZXft5wH192x3s2iRpIhjEJUmtZI62mnPDZGOS3Ul2z8zMDLgsSRoOg7gkadAOJzkHoHs90rUfBM7v224l8MBcX6CqtlbVdFVNT01NDbRYSRoWg7gkadB2Ahu65Q3ArX3t65OckmQ1sAa4vUF9ktTE8tYFSJLGR5L3AJcCZyY5CFwHbAF2JLkauBe4CqCq9iTZAewFjgLXVNWxJoVLUgMGcUnSoqmqNxxn1drjbL8Z2Dy4iiRp6XJoiiRJktSAQVySJElqwCAuSZIkNWAQlyRJkhowiEuSJEkNGMQlSZKkBgzikiRJUgMGcUmSJKkBg7gkSZLUgEFckiRJasAgLkmSJDVgEJckSZIaWFAQT/JjSfYkuTPJe5I8M8mKJLclubt7Pb1v+2uT7E9yV5LLFl6+JEmSNJrmHcSTnAe8DZiuqhcBy4D1wCZgV1WtAXZ170lyUbf+YuBy4IYkyxZWviRJkjSaFjo0ZTnwrCTLgVOBB4B1wLZu/Tbgym55HbC9qh6tqnuA/cAlCzy+JEmSNJLmHcSr6n7g54B7gUPAI1X1IeDsqjrUbXMIOKvb5Tzgvr4vcbBr+xpJNibZnWT3zMzMfEuUJEmSlqyFDE05nV4v92rgXOC0JG98sl3maKu5NqyqrVU1XVXTU1NT8y1RkiRJWrIWMjTl1cA9VTVTVV8Bfht4JXA4yTkA3euRbvuDwPl9+6+kN5RFkiRJmjgLCeL3Aq9IcmqSAGuBfcBOYEO3zQbg1m55J7A+ySlJVgNrgNsXcHxJkiRpZC2f745V9fEktwCfAo4Cnwa2As8GdiS5ml5Yv6rbfk+SHcDebvtrqurYAuuXJEmSRtK8gzhAVV0HXDer+VF6veNzbb8Z2LyQY0qSJEnjwJk1JUmSpAYM4pIkSVIDBnFJkiSpAYO4JEmS1IBBXJIkSWrAIC5JkiQ1YBCXJA1Fkh9LsifJnUnek+SZSVYkuS3J3d3r6a3rlKRhMYhLkgYuyXnA24DpqnoRsAxYD2wCdlXVGmBX916SJoJBXJI0LMuBZyVZDpwKPACsA7Z167cBVzaqTZKGziAuSRq4qrof+DngXuAQ8EhVfQg4u6oOddscAs5qV6UkDZdBXJI0cN3Y73XAauBc4LQkbzyJ/Tcm2Z1k98zMzKDKlKShMohLkobh1cA9VTVTVV8Bfht4JXA4yTkA3euRuXauqq1VNV1V01NTU0MrWpIGySAuSRqGe4FXJDk1SYC1wD5gJ7Ch22YDcGuj+iRp6Ja3LkCSNP6q6uNJbgE+BRwFPg1sBZ4N7EhyNb2wflW7KiVpuAzikqShqKrrgOtmNT9Kr3dckiaOQ1MkSZKkBgzikiRJUgMGcUmSJKkBg7gkSZLUgEFckiRJasAgLkmSJDVgEJckSZIaMIhLkiRJDRjEJUmSpAYM4pIkSVIDBnFJkiSpgeWtC5AkSdLgrNr0/q9pO7DligaVaDZ7xCVJkqQG7BHv42+MkiRJGhZ7xCVJkqQGFhTEkzwvyS1JPptkX5JvS7IiyW1J7u5eT+/b/tok+5PcleSyhZcvSZIkjaaF9ohfD3ygql4IvBjYB2wCdlXVGmBX954kFwHrgYuBy4Ebkixb4PElSZKkkTTvIJ7kOcB3ADcBVNU/VtXDwDpgW7fZNuDKbnkdsL2qHq2qe4D9wCXzPb4kSZI0yhbSI/4CYAa4Ocmnk9yY5DTg7Ko6BNC9ntVtfx5wX9/+B7s2SZIkaeIsJIgvB14G/GpVvRT4O7phKMeROdpqzg2TjUl2J9k9MzOzgBIlSZKkpWkhQfwgcLCqPt69v4VeMD+c5ByA7vVI3/bn9+2/Enhgri9cVVurarqqpqemphZQoiRJkrQ0zTuIV9WDwH1JLuya1gJ7gZ3Ahq5tA3Brt7wTWJ/klCSrgTXA7fM9viRJkjTKFjqhzw8D70ryDODzwFvohfsdSa4G7gWuAqiqPUl20AvrR4FrqurYAo8vSZIkjaQFBfGqugOYnmPV2uNsvxnYvJBjSpIkTQJn/B5/zqwpSZIkNWAQlyRJkhowiEuSJEkNGMQlSUOR5HlJbkny2ST7knxbkhVJbktyd/d6eus6JWlYDOKSpGG5HvhAVb0QeDGwj95EcLuqag2wiyefGE6SxopBXJI0cEmeA3wHcBNAVf1jVT0MrAO2dZttA65sU6EkDZ9BXJI0DC8AZoCbk3w6yY1JTgPOrqpDAN3rWXPtnGRjkt1Jds/MzAyvakkaIIO4JGkYlgMvA361ql4K/B0nMQylqrZW1XRVTU9NTQ2qRkkaKoO4JGkYDgIHq+rj3ftb6AXzw0nOAehejzSqT5KGziAuSRq4qnoQuC/JhV3TWmAvsBPY0LVtAG5tUJ4kNbGgKe4lSToJPwy8K8kzgM8Db6HXIbQjydXAvcBVDeuTpKEyiEuShqKq7gCm51i1dti1SNJS4NAUSZIkqQGDuCRJktSAQVySJElqwCAuSZIkNWAQlyRJkhowiEuSJEkNGMQlSZKkBnyO+FNYten9T3h/YMsVjSqRJEnSOLFHXJIkSWrAIC5JkiQ1YBCXJEmSGjCIS5IkSQ0YxCVJkqQGDOKSJElSAwZxSZIkqQGDuCRJktSAQVySJElqwCAuSZIkNeAU95IkSSNi1ab3ty5Bi2jBPeJJliX5dJLf796vSHJbkru719P7tr02yf4kdyW5bKHHliRJkkbVYgxN+RFgX9/7TcCuqloD7Orek+QiYD1wMXA5cEOSZYtwfEmSJGnkLCiIJ1kJXAHc2Ne8DtjWLW8Druxr315Vj1bVPcB+4JKFHF+SJEkaVQvtEf8l4CeAx/razq6qQwDd61ld+3nAfX3bHezavkaSjUl2J9k9MzOzwBIlSZKkpWfeN2smeQ1wpKo+meTSE9lljraaa8Oq2gpsBZienp5zG0mSpHHijZiTZyFPTXkV8Nok/xp4JvCcJL8JHE5yTlUdSnIOcKTb/iBwft/+K4EHFnB8SZIkaWTNe2hKVV1bVSurahW9mzD/uKreCOwENnSbbQBu7ZZ3AuuTnJJkNbAGuH3elUuSJEkjbBDPEd8C7EhyNXAvcBVAVe1JsgPYCxwFrqmqYwM4viRJkrTkLUoQr6oPAx/ulv8aWHuc7TYDmxfjmJKk0dM9tnY3cH9VvSbJCuC9wCrgAPD6qnqoXYWSNDxOcS9JGqYTmntCkiaBQVySNBQnOfeEJI09g7gkaVhOZu6JJ3B+CUnjyCAuSRq4/rkn5rN/VW2tqumqmp6amlrk6iSpjUE8NUWSpNlOdu4JSRp79ohLkgZuHnNPSNLYs0f8JM01/eyBLVc0qESSxsKcc09I0iQwiEuShupE556QRtXsTjs77HQ8Dk2RJEmSGjCIS5IkSQ0YxCVJkqQGDOKSJElSAwZxSZIkqQGDuCRJktSAQVySJElqwCAuSZIkNeCEPpIkSQPkrNw6HnvEJUmSpAbsEZckSRqyuXrJNXnsEZckSZIaMIhLkiRJDRjEJUmSpAYM4pIkSVIDBnFJkiSpAYO4JEmS1IBBXJIkSWrAIC5JkiQ1YBCXJEmSGjCIS5IkSQ0YxCVJkqQGlrcuYBys2vT+J7w/sOWKRpVIkiRpVMw7iCc5H/gN4OuBx4CtVXV9khXAe4FVwAHg9VX1ULfPtcDVwDHgbVX1wQVVv0TNDuZgOJckSdITLWRoylHgx6vqG4FXANckuQjYBOyqqjXAru493br1wMXA5cANSZYtpHhJkiRpVM27R7yqDgGHuuUvJ9kHnAesAy7tNtsGfBh4e9e+vaoeBe5Jsh+4BPjofGsYZfaaS5IkTbZFGSOeZBXwUuDjwNldSKeqDiU5q9vsPOBjfbsd7Nrm+nobgY0AF1xwwWKUOBIcay5pXM1nOKMkjbsFPzUlybOB3wJ+tKr+5sk2naOt5tqwqrZW1XRVTU9NTS20RElSeyc1nFGSJsGCesSTPJ1eCH9XVf1213w4yTldb/g5wJGu/SBwft/uK4EHFnL8UTLXUBRJmhTzGM4oSWNv3j3iSQLcBOyrql/oW7UT2NAtbwBu7Wtfn+SUJKuBNcDt8z2+JGk0PdlwRuCs4+8pSeNlIT3irwLeBPxlkju6tp8EtgA7klwN3AtcBVBVe5LsAPbS+xPlNVV1bAHHlySNmNnDGXt9Oie030TeOyRpvC3kqSl/ytzjvgHWHmefzcDm+R5TkjS6TnI44xNU1VZgK8D09PSc9xdJ0qhxintJ0sDNYzijJI09p7iXJA3DSQ1nlKRJYBCXJA3cfIYzStK4c2iKJEmS1IBBXJIkSWrAoSmSJEl87eR7B7Zc0agSTQp7xCVJkqQGDOKSJElSAwZxSZIkqYGxHCM+e4yXJEmStNSMZRCXJEmTYa7Ot7luslysGzHt7NNicmiKJEmS1IBBXJIkSWrAIC5JkiQ1YBCXJEmSGjCIS5IkSQ341BRJkgQ4xftsPiFFg2aPuCRJktSAPeKSJGlknEgvtT3ZGhUGcUmSpAnjMKSlwaEpkiRJUgP2iEuStITZcymNL3vEJUmSpAbsEZckSXOa66bHuXrkB9Vr702XGnf2iEuSJEkN2CMuSZIW1Yn2pJ/IftI4M4gvYfP9QSZJkqSlzyAuSdIEWGpPX2nd+936+KNgqX3PjCODuCRJCzSfwGIQ1FIy3xlLh3nz7jjyZk1JkiSpgaH3iCe5HLgeWAbcWFVbhl3DKDuR31j9zVPSqPHaMHzz7ZEf9n7SOBtqEE+yDPjfwHcDB4FPJNlZVXuHWcckWqw/mw5yP0mTyWuDpEk17B7xS4D9VfV5gCTbgXWAP2wX0WKO83qq/QzYkhbBUK4N8/2L4nx6cgfZ+3siP7/tfdY4aJ05hnH8YQfx84D7+t4fBF4+5Bp0HIO82IziRWGY/+AX88I6ir8ctf4rSuvjy2uDpMmUqhrewZKrgMuq6q3d+zcBl1TVD8/abiOwsXt7IXDXSR7qTOCLCyx3lHi+483zHR3Pr6qp1kWMmhO5NizCdWEcjPK/jcXiZ+BnAKP3GRz32jDsHvGDwPl971cCD8zeqKq2Alvne5Aku6tqer77jxrPd7x5vpoAT3ltWOh1YRz4b8PPAPwMYLw+g2E/vvATwJokq5M8A1gP7BxyDZKkpcVrg6SJNNQe8ao6muSHgA/Se0TVO6tqzzBrkCQtLV4bJE2qoT9HvKr+APiDAR9m0v586fmON89XY29I14ZR578NPwPwM4Ax+gyGerOmJEmSpB6nuJckSZIaMIhLkiRJDRjEJUmSpAaGfrPmYkvyQnpTIZ8HFL1nz+6sqn1NCxugJKE3JXT/Od9eYzrg3/P1fCVJk2ucrxMjfbNmkrcDbwC205sQAnoTQawHtlfVlla1DUqS7wFuAO4G7u+aVwLfAPynqvpQq9oGwfMFPF9p4iR5LnAtcCXw+Ix8R4BbgS1V9XCr2oZpnAPYiZr0z2DcrxOjHsT/Cri4qr4yq/0ZwJ6qWtOmssFJsg/4V1V1YFb7auAPquobmxQ2IJ7vP7V7vtIESfJB4I+BbVX1YNf29cAG4NVV9d0t6xuGcQ9gJ8LPYPyvE6M+NOUx4FzgC7Paz+nWjaPlfLX3v9/9wNOHXMsweL49nq80WVZV1Tv6G7pA/o4kP9iopmG7nt4vHQf6Gx8PYMBIB7AT5Gcw5teJUQ/iPwrsSnI3cF/XdgG93xR/qFlVg/VO4BNJtvPVcz6f3nCcm5pVNTier+crTaIvJPkJej3ihwGSnA28ma/+Wxl3Yx3ATpCfwZhfJ0Z6aApAkqfx1bFTofcN+4mqOta0sAFKchHwWp54zjuram/TwgbE8/V8pUmT5HRgE72HEZxNb2zwYWAn8I6q+lLD8oYiybXA6+ndBzY7gO2oqv/RqrZh8TPoGefrxMgHcUmSxl2Sb6fX6fSXkzAu+HHjHMBOVJJv5KtPh5vIz2CcGcRHzKTdSe/5Ap6vNHGS3F5Vl3TLbwWuAX4X+B7g98bxqWDSXMb9OuGEPqNnB/AQcGlVnVFVZwD/EngYeF/TygbD8/V8pUnUP/733wPfU1U/Sy+If3+bkoYryXOTbEny2SR/3f23r2t7Xuv6hiHJ5X3Lz01yY5LPJHl3d8/AJBjr64Q94iMmyV1VdeHJrhtVnu+JrRtVk3a+0olK8hfApfQ6zD5YVdN96z5dVS9tVduwPMkjHN8MrJ2QRzh+qqpe1i3fCDwI/BrwOuA7q+rKlvUNw7hfJ+wRHz1fSPIT/b8JJzm7m9xoHO+k93w9X2kSPRf4JLAbWNEFUJI8m9444Umwqqre8XgIh94jHLthORc0rKuV6ar66ar6QlX9IrCqdUFDMtbXCYP46Pk+4AzgT5I8lORLwIeBFfTurB43s8/3IXrnewaTcb6T9v933M9XOiFVtaqqXlBVq7vXx8PoY8D3tqxtiMY6gJ2gs5L85yQ/Djynm2XzcZOS4cb6OuHQlBGU5IX0Ztb6WFX9bV/75VX1gXaVDUeS/1tVb2pdxyAkeTnw2ap6JMmp9B5f9jJgD/Dfq+qRpgUusvRmwX0DcH9V/VGS7wdeCewFts6eNVfS5Jj1CMezuubHH+G4paoealXbsCS5blbTDVU10/2F5H9W1Q+0qGvYxjn3GMRHTJK30bt7fh/wEuBHqurWbt0/jSUbF0l2ztH8XfTGDVJVrx1uRYOVZA/w4qo6mmQr8HfAbwFru/bXNS1wkSV5F70JK54FPAKcBvwOvfNNVW1oWJ6kJSrJW6rq5tZ1tDQpn8G4555Rn1lzEv074Fuq6m+TrAJuSbKqqq5nPMcNrqTXO3ojvQktAnwr8PMtixqgp1XV0W55uu8HzJ8muaNVUQP0TVX1zUmW05sp7tyqOpbkN4G/aFybpKXrZ4GxD6FPYVI+g7HOPQbx0bPs8T/LVNWBJJfS+6Z8PmPwDTmHaeBHgJ8C/ktV3ZHkH6rqTxrXNSh39vVy/EWS6araneSfA+M4TONp3fCU04BT6d2g9iXgFCZn+mZJc0jymeOtojfb6NjzMwDGPPcYxEfPg0leUlV3AHS/Ib4GeCfwTW1LW3xV9Rjwi0ne170eZry/b98KXJ/kp4EvAh9Nch+9G5Pe2rSywbgJ+CywjN4vW+9L8nngFfSmdJY0uc4GLqP3DOl+Af58+OU04Wcw5rnHMeIjJslK4Gj/45z61r2qqv6sQVlDk+QK4FVV9ZOtaxmkJF8HvIDeLx0Hq+pw45IGJsm5AFX1QDdJx6uBe6vq9raVSWopyU3AzVX1p3Ose3dV/dsGZQ2Vn8H45x6DuCRJktTApDyDUpIkSVpSDOKSJElSAwZxSZIkqQGDuCRJktSAQVySJElq4P8DB0s/cbat0a8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Histogram for spam and ham\n",
    "messages.hist(column='length',by='label',bins=60,figsize=(12,4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the sequences of characters of the messages in sequences of numbers like vectors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split a message into its words and return a list and remove some unuseful words\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation\n",
    "mess = 'Sample message! Notice: it has punctuation.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if not in string.punctuation\n",
    "nopunc = [c for c in mess if c not in string.punctuation]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove stopwords that are not very helpful\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "nopunc = ''.join(nopunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sample message Notice it has punctuation'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nopunc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Sample', 'message', 'Notice', 'it', 'has', 'punctuation']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nopunc.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_mess = [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Sample', 'message', 'Notice', 'punctuation']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_mess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "def text_process(mess):\n",
    "    # 1. Remove punctuation\n",
    "    nopunc = [char for char in mess if char not in string.punctuation]\n",
    "    \n",
    "    nopunc = ''.join(nopunc)\n",
    "    \n",
    "    return [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [Go, jurong, point, crazy, Available, bugis, n...\n",
       "1                       [Ok, lar, Joking, wif, u, oni]\n",
       "2    [Free, entry, 2, wkly, comp, win, FA, Cup, fin...\n",
       "3        [U, dun, say, early, hor, U, c, already, say]\n",
       "4    [Nah, dont, think, goes, usf, lives, around, t...\n",
       "Name: message, dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenize the message\n",
    "messages['message'].head(5).apply(text_process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11425\n"
     ]
    }
   ],
   "source": [
    "# Create a matrix to extract data\n",
    "bow_transformer = CountVectorizer(analyzer=text_process).fit(messages['message'])\n",
    "\n",
    "# Print total number of vocab words\n",
    "print(len(bow_transformer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "U dun say so early hor... U c already then say...\n"
     ]
    }
   ],
   "source": [
    "# Get one message and see vector representation\n",
    "message4 = messages['message'][3]\n",
    "print(message4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 4068)\t2\n",
      "  (0, 4629)\t1\n",
      "  (0, 5261)\t1\n",
      "  (0, 6204)\t1\n",
      "  (0, 6222)\t1\n",
      "  (0, 7186)\t1\n",
      "  (0, 9554)\t2\n",
      "(1, 11425)\n"
     ]
    }
   ],
   "source": [
    "bow4 = bow_transformer.transform([message4])\n",
    "print(bow4)\n",
    "print(bow4.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are 7 unique words in this message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "U\n",
      "say\n"
     ]
    }
   ],
   "source": [
    "print(bow_transformer.get_feature_names()[4068])\n",
    "print(bow_transformer.get_feature_names()[9554])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "U and say happen twice each"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check out how the bag-of-words counts for the entire SMS corpus is a large, sparse matrix:\n",
    "messages_bow = bow_transformer.transform(messages['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Sparse Matrix:  (5572, 11425)\n",
      "Amount of Non-Zero occurences:  50548\n"
     ]
    }
   ],
   "source": [
    "print('Shape of Sparse Matrix: ', messages_bow.shape)\n",
    "print('Amount of Non-Zero occurences: ', messages_bow.nnz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sparsity: 0\n"
     ]
    }
   ],
   "source": [
    "sparsity = (100.0 * messages_bow.nnz / (messages_bow.shape[0] * messages_bow.shape[1]))\n",
    "print('sparsity: {}'.format(round(sparsity)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 9554)\t0.5385626262927564\n",
      "  (0, 7186)\t0.4389365653379857\n",
      "  (0, 6222)\t0.3187216892949149\n",
      "  (0, 6204)\t0.29953799723697416\n",
      "  (0, 5261)\t0.29729957405868723\n",
      "  (0, 4629)\t0.26619801906087187\n",
      "  (0, 4068)\t0.40832589933384067\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "tfidf_transformer = TfidfTransformer().fit(messages_bow)\n",
    "tfidf4 = tfidf_transformer.transform(bow4)\n",
    "print(tfidf4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.2800524267409408\n",
      "8.527076498901426\n"
     ]
    }
   ],
   "source": [
    "#Check the tfidf for the letter u in university\n",
    "print(tfidf_transformer.idf_[bow_transformer.vocabulary_['u']])\n",
    "print(tfidf_transformer.idf_[bow_transformer.vocabulary_['university']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5572, 11425)\n"
     ]
    }
   ],
   "source": [
    "# Transform the bag of words\n",
    "messages_tfidf = tfidf_transformer.transform(messages_bow)\n",
    "print(messages_tfidf.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We already have the vectors\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "spam_detect_model = MultinomialNB().fit(messages_tfidf, messages['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicted: ham\n",
      "expected: ham\n"
     ]
    }
   ],
   "source": [
    "# Try the prediction with a single message\n",
    "print('predicted:', spam_detect_model.predict(tfidf4)[0])\n",
    "print('expected:', messages.label[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ham' 'ham' 'spam' ... 'ham' 'ham' 'ham']\n"
     ]
    }
   ],
   "source": [
    "# Perform all the predictions\n",
    "all_predictions = spam_detect_model.predict(messages_tfidf)\n",
    "print(all_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ham       0.98      1.00      0.99      4825\n",
      "        spam       1.00      0.85      0.92       747\n",
      "\n",
      "    accuracy                           0.98      5572\n",
      "   macro avg       0.99      0.92      0.95      5572\n",
      "weighted avg       0.98      0.98      0.98      5572\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print (classification_report(messages['label'], all_predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4457 1115 5572\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "msg_train, msg_test, label_train, label_test = \\\n",
    "train_test_split(messages['message'], messages['label'], test_size=0.2)\n",
    "\n",
    "print(len(msg_train), len(msg_test), len(msg_train) + len(msg_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All the previous steps resumed in 3 lones of code thanks to the sklearn pipeline\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('bow', CountVectorizer(analyzer=text_process)),  # strings to token integer counts\n",
    "    ('tfidf', TfidfTransformer()),  # integer counts to weighted TF-IDF scores\n",
    "    ('classifier', MultinomialNB()),  # train on TF-IDF vectors w/ Naive Bayes classifier\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('bow',\n",
       "                 CountVectorizer(analyzer=<function text_process at 0x0000017AFF9EA0D0>)),\n",
       "                ('tfidf', TfidfTransformer()),\n",
       "                ('classifier', MultinomialNB())])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(msg_train,label_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform the prediction\n",
    "predictions = pipeline.predict(msg_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ham       1.00      0.95      0.98       998\n",
      "        spam       0.71      1.00      0.83       117\n",
      "\n",
      "    accuracy                           0.96      1115\n",
      "   macro avg       0.85      0.98      0.90      1115\n",
      "weighted avg       0.97      0.96      0.96      1115\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Get the report\n",
    "print(classification_report(predictions,label_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}