ANN_TelecomChurn_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Develop a Deep Learning Based Churn Prediction Engine\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load & understand the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerID</th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>MultipleLines</th>\n",
       "      <th>InternetService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>...</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>Contract</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>PaymentMethod</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7590-VHVEG</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5575-GNVDE</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>34</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3668-QPYBK</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7795-CFOCW</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>45</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9237-HQITU</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   customerID  gender  SeniorCitizen Partner Dependents  tenure PhoneService  \\\n",
       "0  7590-VHVEG  Female              0     Yes         No       1           No   \n",
       "1  5575-GNVDE    Male              0      No         No      34          Yes   \n",
       "2  3668-QPYBK    Male              0      No         No       2          Yes   \n",
       "3  7795-CFOCW    Male              0      No         No      45           No   \n",
       "4  9237-HQITU  Female              0      No         No       2          Yes   \n",
       "\n",
       "  MultipleLines InternetService OnlineSecurity  ... DeviceProtection  \\\n",
       "0            No             DSL             No  ...               No   \n",
       "1            No             DSL            Yes  ...              Yes   \n",
       "2            No             DSL            Yes  ...               No   \n",
       "3            No             DSL            Yes  ...              Yes   \n",
       "4            No     Fiber optic             No  ...               No   \n",
       "\n",
       "  TechSupport StreamingTV StreamingMovies        Contract PaperlessBilling  \\\n",
       "0          No          No              No  Month-to-month              Yes   \n",
       "1          No          No              No        One year               No   \n",
       "2          No          No              No  Month-to-month              Yes   \n",
       "3         Yes          No              No        One year               No   \n",
       "4          No          No              No  Month-to-month              Yes   \n",
       "\n",
       "               PaymentMethod MonthlyCharges  TotalCharges Churn  \n",
       "0           Electronic check          29.85         29.85    No  \n",
       "1               Mailed check          56.95        1889.5    No  \n",
       "2               Mailed check          53.85        108.15   Yes  \n",
       "3  Bank transfer (automatic)          42.30       1840.75    No  \n",
       "4           Electronic check          70.70        151.65   Yes  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the data\n",
    "data = pd.read_csv(\"TelcoChurn1.csv\")\n",
    "\n",
    "# Look at a snapshot of data\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>tenure</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7043.000000</td>\n",
       "      <td>7043.000000</td>\n",
       "      <td>7043.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.162147</td>\n",
       "      <td>32.371149</td>\n",
       "      <td>64.761692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.368612</td>\n",
       "      <td>24.559481</td>\n",
       "      <td>30.090047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>35.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>70.350000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>89.850000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>118.750000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       SeniorCitizen       tenure  MonthlyCharges\n",
       "count    7043.000000  7043.000000     7043.000000\n",
       "mean        0.162147    32.371149       64.761692\n",
       "std         0.368612    24.559481       30.090047\n",
       "min         0.000000     0.000000       18.250000\n",
       "25%         0.000000     9.000000       35.500000\n",
       "50%         0.000000    29.000000       70.350000\n",
       "75%         0.000000    55.000000       89.850000\n",
       "max         1.000000    72.000000      118.750000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# See the summary stats and frequency distribution of features\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "customerID          0\n",
      "gender              0\n",
      "SeniorCitizen       0\n",
      "Partner             0\n",
      "Dependents          0\n",
      "tenure              0\n",
      "PhoneService        0\n",
      "MultipleLines       0\n",
      "InternetService     0\n",
      "OnlineSecurity      0\n",
      "OnlineBackup        0\n",
      "DeviceProtection    0\n",
      "TechSupport         0\n",
      "StreamingTV         0\n",
      "StreamingMovies     0\n",
      "Contract            0\n",
      "PaperlessBilling    0\n",
      "PaymentMethod       0\n",
      "MonthlyCharges      0\n",
      "TotalCharges        0\n",
      "Churn               0\n",
      "dtype: int64\n",
      "(array([ 488,  753,  936, 1082, 1340, 3331, 3826, 4380, 5218, 6670, 6754]), array([19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19]))\n"
     ]
    }
   ],
   "source": [
    "# Lets' see if there are any missing values\n",
    "\n",
    "print (data.apply(lambda x: sum(x.isnull()),axis=0))\n",
    "print (np.where(data.applymap(lambda x: x == ' ')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yes    1869\n",
      "No     5174\n",
      "Name: Churn, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Let's see if there is class imbalance in the target variable\n",
    "print (data['Churn'].value_counts(ascending=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['TotalCharges'].replace(to_replace = ' ', value= np.nan, inplace = True)\n",
    "data['TotalCharges'] = data['TotalCharges'].astype(float)\n",
    "data.dropna(axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Observations:\n",
    "\n",
    "**On Type conversions:**\n",
    "\n",
    "- Columns like CustomerID can be removed from the analysis\n",
    "- We see that 'Tenure' and 'MonthlyCharges' are numeric columns present in the data, with the data close to normal distribution. \n",
    "- Along with them, 'TotalCharges' is also a numeric column but contains some info missing, but still is not a nan.\n",
    "- The column 'SeniorCitizen' is a categorical column by its nature with 'Yes' as 1, and No as 0. So it shuold be converted into Categorical type\n",
    "- All the categorical attribtues are strings. Hence there is need to convert them into numbers, by a way of encoding.\n",
    "- Among the categorical attribtues, majority of them have binary classes(2 levels). Label encoding would help assign labels 0,1 for the levels as appropriate.\n",
    "- But attributes like 'PaymentMethod', 'Contract', 'InternetService' are nominal and have more than 2 levels. So along with label encoding, we need to convert them into equidistant levels.\n",
    "\n",
    "**On Missingness of data:**\n",
    " The data is clean and there are no missing values in the data\n",
    " \n",
    "**On the class imbalance in the target attribute**\n",
    "There are more instances where the customers din't churn than those that have custoemrs churned out. Class imbalance is clearly seen."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the data into train and test sets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5625, 20) (1407, 20)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "y = data['Churn']\n",
    "X = data.loc[:, data.columns != 'Churn']\n",
    "\n",
    "X_train, X_test, y_train, y_test =   train_test_split(X, y, test_size=0.20, random_state=111)\n",
    "\n",
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Remove customerID\n",
    "X_train.drop(['customerID'], axis = 1, inplace=True)\n",
    "X_test.drop(['customerID'], axis = 1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num missing values before imputation:\n",
      "TotalCharges    0\n",
      "dtype: int64\n",
      "Num missing values before imputation:\n",
      "TotalCharges    0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Impute missing values, if any!. Check number of missing values\n",
    "print(\"Num missing values before imputation:\")\n",
    "print(pd.DataFrame(X_train['TotalCharges']).isnull().sum())\n",
    "\n",
    "print(\"Num missing values before imputation:\")\n",
    "print(pd.DataFrame(X_test['TotalCharges']).isnull().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Type Conversions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert 'SeniorCitizen' column into categorical\n",
    "X_train['SeniorCitizen']=pd.Categorical(X_train['SeniorCitizen'])\n",
    "X_test['SeniorCitizen']=pd.Categorical(X_test['SeniorCitizen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5625,) (1407,)\n"
     ]
    }
   ],
   "source": [
    "# Encode target variables to 0, 1\n",
    "y_train = y_train.map(dict(Yes=1, No=0))\n",
    "y_test = y_test.map(dict(Yes = 1, No=0))\n",
    "print(y_train.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standardizing numeric attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Divide the columns into 3 categories, one ofor standardisation, one for label encoding and one for one hot encoding\n",
    "num_cols = [\"tenure\", 'MonthlyCharges', 'TotalCharges']\n",
    "cat_cols_ohe =['PaymentMethod', 'Contract', 'InternetService'] # those that need one-hot encoding\n",
    "cat_cols_le = list(set(X_train.columns)- set(num_cols) - set(cat_cols_ohe)) #those that need label encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler= StandardScaler()\n",
    "\n",
    "X_train[num_cols] = scaler.fit_transform(X_train[num_cols])\n",
    "X_test[num_cols] = scaler.transform(X_test[num_cols])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.DataFrame(X_train)\n",
    "X_test= pd.DataFrame(X_test)\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "le = LabelEncoder()\n",
    "ohe = OneHotEncoder()\n",
    "\n",
    "for col in cat_cols_le:\n",
    "    le.fit(X_train[col])\n",
    "    X_train[col] = le.transform(X_train[col])\n",
    "    X_test[col] = le.transform(X_test[col])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5625, 19) (1407, 19)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5625, 26) (1407, 26)\n"
     ]
    }
   ],
   "source": [
    "ohe.fit(X_train[cat_cols_ohe])\n",
    "tr_cols= ohe.transform(X_train[cat_cols_ohe])\n",
    "te_cols = ohe.transform(X_test[cat_cols_ohe])\n",
    "\n",
    "X_train.drop(columns=cat_cols_ohe, inplace=True)\n",
    "X_test.drop(columns=cat_cols_ohe, inplace=True)\n",
    "\n",
    "X_train = np.hstack((X_train,tr_cols.toarray()))\n",
    "X_test = np.hstack((X_test, te_cols.toarray()))\n",
    "print(X_train.shape, X_test.shape)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5625,) (1407,)\n"
     ]
    }
   ],
   "source": [
    "print(y_train.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    4148\n",
      "1    1477\n",
      "Name: Churn, dtype: int64 \n",
      " 0    1015\n",
      "1     392\n",
      "Name: Churn, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y_train.value_counts(), '\\n', y_test.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the ANN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing necessary modules\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras import Input, Model\n",
    "from tensorflow.keras.layers import Dense, Dropout, Activation\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 7\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X_train = np.array(X_train)\n",
    "y_train = np.array(y_train)\n",
    "X_test = np.array(X_test)\n",
    "y_test = np.array(y_test)\n",
    "\n",
    "input_shape = X_train.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0716 04:06:54.175837 140735948321664 nn_ops.py:4372] Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "W0716 04:06:54.252901 140735948321664 data_adapter.py:1091] sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "W0716 04:06:54.320842 140735948321664 nn_ops.py:4372] Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5625 samples\n",
      "Epoch 1/50\n",
      "5625/5625 [==============================] - 1s 201us/sample - loss: 0.2010 - accuracy: 0.6101\n",
      "Epoch 2/50\n",
      "5625/5625 [==============================] - 0s 84us/sample - loss: 0.1800 - accuracy: 0.6910\n",
      "Epoch 3/50\n",
      "5625/5625 [==============================] - 0s 87us/sample - loss: 0.1766 - accuracy: 0.6928\n",
      "Epoch 4/50\n",
      "5625/5625 [==============================] - 1s 89us/sample - loss: 0.1742 - accuracy: 0.6921\n",
      "Epoch 5/50\n",
      "5625/5625 [==============================] - 1s 102us/sample - loss: 0.1736 - accuracy: 0.6946\n",
      "Epoch 6/50\n",
      "5625/5625 [==============================] - 1s 95us/sample - loss: 0.1751 - accuracy: 0.7017\n",
      "Epoch 7/50\n",
      "5625/5625 [==============================] - 1s 98us/sample - loss: 0.1728 - accuracy: 0.7056\n",
      "Epoch 8/50\n",
      "5625/5625 [==============================] - 0s 87us/sample - loss: 0.1740 - accuracy: 0.6994\n",
      "Epoch 9/50\n",
      "5625/5625 [==============================] - 1s 91us/sample - loss: 0.1702 - accuracy: 0.7067\n",
      "Epoch 10/50\n",
      "5625/5625 [==============================] - 0s 86us/sample - loss: 0.1718 - accuracy: 0.7010\n",
      "Epoch 11/50\n",
      "5625/5625 [==============================] - 0s 89us/sample - loss: 0.1737 - accuracy: 0.7003\n",
      "Epoch 12/50\n",
      "5625/5625 [==============================] - 0s 87us/sample - loss: 0.1721 - accuracy: 0.7122\n",
      "Epoch 13/50\n",
      "5625/5625 [==============================] - 1s 89us/sample - loss: 0.1714 - accuracy: 0.7081\n",
      "Epoch 14/50\n",
      "5625/5625 [==============================] - 0s 87us/sample - loss: 0.1691 - accuracy: 0.7084\n",
      "Epoch 15/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1708 - accuracy: 0.7116\n",
      "Epoch 16/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1692 - accuracy: 0.7115\n",
      "Epoch 17/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1680 - accuracy: 0.7214\n",
      "Epoch 18/50\n",
      "5625/5625 [==============================] - 1s 94us/sample - loss: 0.1690 - accuracy: 0.7188\n",
      "Epoch 19/50\n",
      "5625/5625 [==============================] - 1s 109us/sample - loss: 0.1664 - accuracy: 0.7191\n",
      "Epoch 20/50\n",
      "5625/5625 [==============================] - 1s 94us/sample - loss: 0.1686 - accuracy: 0.7211\n",
      "Epoch 21/50\n",
      "5625/5625 [==============================] - 1s 92us/sample - loss: 0.1661 - accuracy: 0.7211\n",
      "Epoch 22/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1664 - accuracy: 0.7182\n",
      "Epoch 23/50\n",
      "5625/5625 [==============================] - 1s 95us/sample - loss: 0.1681 - accuracy: 0.7202\n",
      "Epoch 24/50\n",
      "5625/5625 [==============================] - 0s 86us/sample - loss: 0.1660 - accuracy: 0.7180\n",
      "Epoch 25/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1664 - accuracy: 0.7193\n",
      "Epoch 26/50\n",
      "5625/5625 [==============================] - 0s 89us/sample - loss: 0.1642 - accuracy: 0.7244\n",
      "Epoch 27/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1667 - accuracy: 0.7193\n",
      "Epoch 28/50\n",
      "5625/5625 [==============================] - 0s 89us/sample - loss: 0.1636 - accuracy: 0.7218\n",
      "Epoch 29/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1624 - accuracy: 0.7239\n",
      "Epoch 30/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1646 - accuracy: 0.7239\n",
      "Epoch 31/50\n",
      "5625/5625 [==============================] - 1s 89us/sample - loss: 0.1614 - accuracy: 0.7228\n",
      "Epoch 32/50\n",
      "5625/5625 [==============================] - 0s 87us/sample - loss: 0.1622 - accuracy: 0.7291\n",
      "Epoch 33/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1627 - accuracy: 0.7234\n",
      "Epoch 34/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1627 - accuracy: 0.7172\n",
      "Epoch 35/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1608 - accuracy: 0.7212\n",
      "Epoch 36/50\n",
      "5625/5625 [==============================] - 0s 87us/sample - loss: 0.1622 - accuracy: 0.7271\n",
      "Epoch 37/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1607 - accuracy: 0.7202\n",
      "Epoch 38/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1606 - accuracy: 0.7211\n",
      "Epoch 39/50\n",
      "5625/5625 [==============================] - 1s 92us/sample - loss: 0.1602 - accuracy: 0.7330\n",
      "Epoch 40/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1598 - accuracy: 0.7262\n",
      "Epoch 41/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1597 - accuracy: 0.7182\n",
      "Epoch 42/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1592 - accuracy: 0.7252\n",
      "Epoch 43/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1575 - accuracy: 0.7303\n",
      "Epoch 44/50\n",
      "5625/5625 [==============================] - 1s 89us/sample - loss: 0.1582 - accuracy: 0.7241\n",
      "Epoch 45/50\n",
      "5625/5625 [==============================] - 1s 94us/sample - loss: 0.1590 - accuracy: 0.7296\n",
      "Epoch 46/50\n",
      "5625/5625 [==============================] - 1s 90us/sample - loss: 0.1603 - accuracy: 0.7209\n",
      "Epoch 47/50\n",
      "5625/5625 [==============================] - 0s 88us/sample - loss: 0.1578 - accuracy: 0.7273\n",
      "Epoch 48/50\n",
      "5625/5625 [==============================] - 0s 86us/sample - loss: 0.1579 - accuracy: 0.7252\n",
      "Epoch 49/50\n",
      "5625/5625 [==============================] - 1s 89us/sample - loss: 0.1578 - accuracy: 0.7287\n",
      "Epoch 50/50\n",
      "5625/5625 [==============================] - 0s 84us/sample - loss: 0.1566 - accuracy: 0.7230\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1476b1908>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(32, input_dim=input_shape, kernel_initializer='uniform', activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dropout(0.7))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "\n",
    "model.fit(X_train, y_train,\n",
    "              epochs=25,\n",
    "          batch_size=24, class_weight={0:0.2, 1:0.8})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1407/1407 [==============================] - 0s 123us/sample - loss: 0.5560 - accuracy: 0.6972\n"
     ]
    }
   ],
   "source": [
    "\n",
    "score = model.evaluate(X_test, y_test, batch_size=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5559938355973195, 0.69722813]\n",
      "Accuracy : 69.723%\n"
     ]
    }
   ],
   "source": [
    "print(score)\n",
    "print (\"Accuracy : %s\" % \"{0:.3%}\".format(score[1]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train_pred_dl=model.predict_classes(X_train)\n",
    "test_pred_dl=model.predict_classes(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[642 373]\n",
      " [ 53 339]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "mlp_conf_matrix = metrics.confusion_matrix(y_test, test_pred_dl)\n",
    "print (mlp_conf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 69.723%\n",
      "Recall : 86.480%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "accuracy = metrics.accuracy_score(y_test,test_pred_dl)\n",
    "    \n",
    "print (\"Accuracy : %s\" % \"{0:.3%}\".format(accuracy))\n",
    "\n",
    "#Print Recall\n",
    "recall = metrics.recall_score(y_test,test_pred_dl)\n",
    "    \n",
    "print (\"Recall : %s\" % \"{0:.3%}\".format(recall))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP using features from AutoEncoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding_dim  = 32\n",
    "# this is our input placeholder\n",
    "input_img = Input(shape=(input_shape,))\n",
    "\n",
    "# \"encoded\" is the encoded representation of the input\n",
    "encoded = Dense(encoding_dim, activation='relu')(input_img)\n",
    "\n",
    "# \"decoded\" is the lossy reconstruction of the input\n",
    "decoded = Dense(input_shape, activation='sigmoid')(encoded)\n",
    "\n",
    "# this model maps an input to its reconstruction\n",
    "autoencoder = Model(inputs=input_img, outputs=decoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy') #optimizer = adam --- can also be used "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5625 samples, validate on 1407 samples\n",
      "Epoch 1/50\n",
      "5625/5625 [==============================] - 1s 140us/sample - loss: 0.7783 - val_loss: 0.7770\n",
      "Epoch 2/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.7766 - val_loss: 0.7752\n",
      "Epoch 3/50\n",
      "5625/5625 [==============================] - 0s 84us/sample - loss: 0.7748 - val_loss: 0.7734\n",
      "Epoch 4/50\n",
      "5625/5625 [==============================] - 0s 81us/sample - loss: 0.7730 - val_loss: 0.7716\n",
      "Epoch 5/50\n",
      "5625/5625 [==============================] - 0s 79us/sample - loss: 0.7711 - val_loss: 0.7697\n",
      "Epoch 6/50\n",
      "5625/5625 [==============================] - 0s 77us/sample - loss: 0.7693 - val_loss: 0.7678\n",
      "Epoch 7/50\n",
      "5625/5625 [==============================] - 0s 81us/sample - loss: 0.7674 - val_loss: 0.7659\n",
      "Epoch 8/50\n",
      "5625/5625 [==============================] - 0s 77us/sample - loss: 0.7655 - val_loss: 0.7640\n",
      "Epoch 9/50\n",
      "5625/5625 [==============================] - 0s 83us/sample - loss: 0.7636 - val_loss: 0.7621\n",
      "Epoch 10/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.7617 - val_loss: 0.7602\n",
      "Epoch 11/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7598 - val_loss: 0.7582\n",
      "Epoch 12/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7579 - val_loss: 0.7563\n",
      "Epoch 13/50\n",
      "5625/5625 [==============================] - 0s 79us/sample - loss: 0.7560 - val_loss: 0.7544\n",
      "Epoch 14/50\n",
      "5625/5625 [==============================] - 0s 79us/sample - loss: 0.7541 - val_loss: 0.7526\n",
      "Epoch 15/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7523 - val_loss: 0.7507\n",
      "Epoch 16/50\n",
      "5625/5625 [==============================] - 0s 80us/sample - loss: 0.7505 - val_loss: 0.7489\n",
      "Epoch 17/50\n",
      "5625/5625 [==============================] - 0s 81us/sample - loss: 0.7486 - val_loss: 0.7470\n",
      "Epoch 18/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.7468 - val_loss: 0.7452\n",
      "Epoch 19/50\n",
      "5625/5625 [==============================] - 0s 79us/sample - loss: 0.7451 - val_loss: 0.7435\n",
      "Epoch 20/50\n",
      "5625/5625 [==============================] - 0s 80us/sample - loss: 0.7433 - val_loss: 0.7417\n",
      "Epoch 21/50\n",
      "5625/5625 [==============================] - 0s 83us/sample - loss: 0.7415 - val_loss: 0.7399\n",
      "Epoch 22/50\n",
      "5625/5625 [==============================] - 1s 92us/sample - loss: 0.7398 - val_loss: 0.7382\n",
      "Epoch 23/50\n",
      "5625/5625 [==============================] - 1s 108us/sample - loss: 0.7381 - val_loss: 0.7364\n",
      "Epoch 24/50\n",
      "5625/5625 [==============================] - 1s 93us/sample - loss: 0.7364 - val_loss: 0.7347\n",
      "Epoch 25/50\n",
      "5625/5625 [==============================] - 0s 80us/sample - loss: 0.7347 - val_loss: 0.7330\n",
      "Epoch 26/50\n",
      "5625/5625 [==============================] - 0s 76us/sample - loss: 0.7330 - val_loss: 0.7313\n",
      "Epoch 27/50\n",
      "5625/5625 [==============================] - 0s 77us/sample - loss: 0.7313 - val_loss: 0.7297\n",
      "Epoch 28/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7297 - val_loss: 0.7280\n",
      "Epoch 29/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.7281 - val_loss: 0.7264\n",
      "Epoch 30/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7265 - val_loss: 0.7247\n",
      "Epoch 31/50\n",
      "5625/5625 [==============================] - 0s 77us/sample - loss: 0.7248 - val_loss: 0.7231\n",
      "Epoch 32/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7232 - val_loss: 0.7215\n",
      "Epoch 33/50\n",
      "5625/5625 [==============================] - 0s 74us/sample - loss: 0.7216 - val_loss: 0.7199\n",
      "Epoch 34/50\n",
      "5625/5625 [==============================] - 0s 81us/sample - loss: 0.7201 - val_loss: 0.7183\n",
      "Epoch 35/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7185 - val_loss: 0.7167\n",
      "Epoch 36/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7169 - val_loss: 0.7152\n",
      "Epoch 37/50\n",
      "5625/5625 [==============================] - 0s 74us/sample - loss: 0.7154 - val_loss: 0.7136\n",
      "Epoch 38/50\n",
      "5625/5625 [==============================] - 0s 74us/sample - loss: 0.7139 - val_loss: 0.7121\n",
      "Epoch 39/50\n",
      "5625/5625 [==============================] - 0s 77us/sample - loss: 0.7124 - val_loss: 0.7106\n",
      "Epoch 40/50\n",
      "5625/5625 [==============================] - 0s 77us/sample - loss: 0.7109 - val_loss: 0.7091\n",
      "Epoch 41/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.7094 - val_loss: 0.7076\n",
      "Epoch 42/50\n",
      "5625/5625 [==============================] - 0s 75us/sample - loss: 0.7079 - val_loss: 0.7061\n",
      "Epoch 43/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.7065 - val_loss: 0.7047\n",
      "Epoch 44/50\n",
      "5625/5625 [==============================] - 0s 74us/sample - loss: 0.7051 - val_loss: 0.7032\n",
      "Epoch 45/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.7036 - val_loss: 0.7018\n",
      "Epoch 46/50\n",
      "5625/5625 [==============================] - 0s 76us/sample - loss: 0.7022 - val_loss: 0.7003\n",
      "Epoch 47/50\n",
      "5625/5625 [==============================] - 0s 76us/sample - loss: 0.7008 - val_loss: 0.6989\n",
      "Epoch 48/50\n",
      "5625/5625 [==============================] - 0s 78us/sample - loss: 0.6994 - val_loss: 0.6975\n",
      "Epoch 49/50\n",
      "5625/5625 [==============================] - 0s 77us/sample - loss: 0.6980 - val_loss: 0.6961\n",
      "Epoch 50/50\n",
      "5625/5625 [==============================] - 0s 80us/sample - loss: 0.6966 - val_loss: 0.6947\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1473b3390>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autoencoder.fit(X_train, X_train,\n",
    "                epochs=50,\n",
    "                batch_size=24,\n",
    "                shuffle=True,\n",
    "                validation_data=(X_test, X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this model maps an input to its encoded representation\n",
    "encoder = Model(inputs=input_img, outputs=encoded)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_encoded = encoder.predict(X_train)\n",
    "x_test_encoded = encoder.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.18937707, 0.14511806, ..., 0.        , 0.        ,\n",
       "        0.6969709 ],\n",
       "       [0.        , 0.        , 0.5015595 , ..., 0.        , 0.        ,\n",
       "        0.1034191 ],\n",
       "       [0.        , 0.21934086, 0.        , ..., 0.        , 0.        ,\n",
       "        0.85755384],\n",
       "       ...,\n",
       "       [0.10965183, 0.5013616 , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.37887457],\n",
       "       [0.        , 0.38840058, 0.        , ..., 0.6153631 , 0.        ,\n",
       "        0.9018389 ]], dtype=float32)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5625, 32)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_encoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1407, 32)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test_encoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2 = Sequential()\n",
    "\n",
    "model2.add(Dense(64, input_dim = 32, kernel_initializer='uniform', activation='relu'))\n",
    "model2.add(Dropout(0.2))\n",
    "model2.add(Dense(16, activation='relu'))\n",
    "model2.add(Dropout(0.2))\n",
    "model2.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2.compile(loss='binary_crossentropy',\n",
    "              optimizer='Adam',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5625, 32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_encoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0716 04:06:25.269119 140735948321664 data_adapter.py:1091] sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5625 samples\n",
      "Epoch 1/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1653 - accuracy: 0.7273\n",
      "Epoch 2/25\n",
      "5625/5625 [==============================] - 0s 64us/sample - loss: 0.1651 - accuracy: 0.7298\n",
      "Epoch 3/25\n",
      "5625/5625 [==============================] - 0s 62us/sample - loss: 0.1643 - accuracy: 0.7278\n",
      "Epoch 4/25\n",
      "5625/5625 [==============================] - 0s 61us/sample - loss: 0.1654 - accuracy: 0.7355\n",
      "Epoch 5/25\n",
      "5625/5625 [==============================] - 0s 61us/sample - loss: 0.1646 - accuracy: 0.7253\n",
      "Epoch 6/25\n",
      "5625/5625 [==============================] - 0s 62us/sample - loss: 0.1646 - accuracy: 0.7250\n",
      "Epoch 7/25\n",
      "5625/5625 [==============================] - 0s 61us/sample - loss: 0.1642 - accuracy: 0.7312\n",
      "Epoch 8/25\n",
      "5625/5625 [==============================] - 0s 61us/sample - loss: 0.1617 - accuracy: 0.7307\n",
      "Epoch 9/25\n",
      "5625/5625 [==============================] - 0s 65us/sample - loss: 0.1630 - accuracy: 0.7282\n",
      "Epoch 10/25\n",
      "5625/5625 [==============================] - 0s 61us/sample - loss: 0.1634 - accuracy: 0.7305\n",
      "Epoch 11/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1621 - accuracy: 0.7308\n",
      "Epoch 12/25\n",
      "5625/5625 [==============================] - 0s 68us/sample - loss: 0.1610 - accuracy: 0.7259\n",
      "Epoch 13/25\n",
      "5625/5625 [==============================] - 0s 62us/sample - loss: 0.1620 - accuracy: 0.7236\n",
      "Epoch 14/25\n",
      "5625/5625 [==============================] - 0s 66us/sample - loss: 0.1615 - accuracy: 0.7337\n",
      "Epoch 15/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1636 - accuracy: 0.7301\n",
      "Epoch 16/25\n",
      "5625/5625 [==============================] - 0s 61us/sample - loss: 0.1623 - accuracy: 0.7330\n",
      "Epoch 17/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1626 - accuracy: 0.7278\n",
      "Epoch 18/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1618 - accuracy: 0.7294\n",
      "Epoch 19/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1627 - accuracy: 0.7225\n",
      "Epoch 20/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1615 - accuracy: 0.7340\n",
      "Epoch 21/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1607 - accuracy: 0.7344\n",
      "Epoch 22/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1612 - accuracy: 0.7276\n",
      "Epoch 23/25\n",
      "5625/5625 [==============================] - 0s 63us/sample - loss: 0.1620 - accuracy: 0.7305\n",
      "Epoch 24/25\n",
      "5625/5625 [==============================] - 0s 62us/sample - loss: 0.1614 - accuracy: 0.7278\n",
      "Epoch 25/25\n",
      "5625/5625 [==============================] - 0s 62us/sample - loss: 0.1594 - accuracy: 0.7403\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x14a313668>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model2.fit(x_train_encoded, y_train, batch_size=32, epochs=25, class_weight={0:0.2, 1:0.8})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1407/1407 [==============================] - 0s 36us/sample - loss: 0.5407 - accuracy: 0.7200\n",
      "[0.5406732543953446, 0.7199716]\n"
     ]
    }
   ],
   "source": [
    "score2 = model2.evaluate(x_test_encoded, y_test)\n",
    "print (score2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train_pred_dlac=model2.predict_classes(x_train_encoded)\n",
    "test_pred_dlac=model2.predict_classes(x_test_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[701 314]\n",
      " [ 80 312]]\n"
     ]
    }
   ],
   "source": [
    "dlac_conf_matrix = metrics.confusion_matrix(y_test, test_pred_dlac)\n",
    "print (dlac_conf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 71.997%\n",
      "Recall : 79.592%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "accuracy = metrics.accuracy_score(y_test,test_pred_dlac)\n",
    "    \n",
    "print (\"Accuracy : %s\" % \"{0:.3%}\".format(accuracy))\n",
    "\n",
    "#Print Recall\n",
    "recall = metrics.recall_score(y_test,test_pred_dlac)\n",
    "    \n",
    "print (\"Recall : %s\" % \"{0:.3%}\".format(recall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}