Logistic_Regression_Consulting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression Consulting \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "\n",
    "\n",
    "\n",
    "The data is :\n",
    "\n",
    "    Name : Name of the latest contact at Company\n",
    "    Age: Customer Age\n",
    "    Total_Purchase: Total Ads Purchased\n",
    "    Account_Manager: Binary 0=No manager, 1= Account manager assigned\n",
    "    Years: Totaly Years as a customer\n",
    "    Num_sites: Number of websites that use the service.\n",
    "    Onboard_date: Date that the name of the latest contact was onboarded\n",
    "    Location: Client HQ Address\n",
    "    Company: Name of Client Company\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName('logregconsult').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = spark.read.csv('customer_churn.csv',inferSchema=True,\n",
    "                     header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Names: string (nullable = true)\n",
      " |-- Age: double (nullable = true)\n",
      " |-- Total_Purchase: double (nullable = true)\n",
      " |-- Account_Manager: integer (nullable = true)\n",
      " |-- Years: double (nullable = true)\n",
      " |-- Num_Sites: double (nullable = true)\n",
      " |-- Onboard_date: timestamp (nullable = true)\n",
      " |-- Location: string (nullable = true)\n",
      " |-- Company: string (nullable = true)\n",
      " |-- Churn: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check out the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "|summary|              Age|   Total_Purchase|   Account_Manager|            Years|         Num_Sites|              Churn|\n",
      "+-------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "|  count|              900|              900|               900|              900|               900|                900|\n",
      "|   mean|41.81666666666667|10062.82403333334|0.4811111111111111| 5.27315555555555| 8.587777777777777|0.16666666666666666|\n",
      "| stddev|6.127560416916251|2408.644531858096|0.4999208935073339|1.274449013194616|1.7648355920350969| 0.3728852122772358|\n",
      "|    min|             22.0|            100.0|                 0|              1.0|               3.0|                  0|\n",
      "|    max|             65.0|         18026.01|                 1|             9.15|              14.0|                  1|\n",
      "+-------+-----------------+-----------------+------------------+-----------------+------------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Names',\n",
       " 'Age',\n",
       " 'Total_Purchase',\n",
       " 'Account_Manager',\n",
       " 'Years',\n",
       " 'Num_Sites',\n",
       " 'Onboard_date',\n",
       " 'Location',\n",
       " 'Company',\n",
       " 'Churn']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Format for MLlib\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assembler = VectorAssembler(inputCols=['Age',\n",
    " 'Total_Purchase',\n",
    " 'Account_Manager',\n",
    " 'Years',\n",
    " 'Num_Sites'],outputCol='features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output = assembler.transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_data = output.select('features','churn')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Train Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_churn,test_churn = final_data.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_churn = LogisticRegression(labelCol='churn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fitted_churn_model = lr_churn.fit(train_churn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_sum = fitted_churn_model.summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------------+-------------------+\n",
      "|summary|              churn|         prediction|\n",
      "+-------+-------------------+-------------------+\n",
      "|  count|                632|                632|\n",
      "|   mean|0.16772151898734178|0.13924050632911392|\n",
      "| stddev|0.37391474020622584| 0.3464715405857694|\n",
      "|    min|                  0|                0.0|\n",
      "|    max|                  1|                1.0|\n",
      "+-------+-------------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "training_sum.predictions.describe().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate results\n",
    "\n",
    "Let's evaluate the results on the data set we were given (using the test data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_and_labels = fitted_churn_model.evaluate(test_churn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+--------------------+--------------------+----------+\n",
      "|            features|churn|       rawPrediction|         probability|prediction|\n",
      "+--------------------+-----+--------------------+--------------------+----------+\n",
      "|[29.0,11274.46,1....|    0|[4.87277048314045...|[0.99240597473215...|       0.0|\n",
      "|[30.0,8403.78,1.0...|    0|[6.62706699787450...|[0.99867770995491...|       0.0|\n",
      "|[30.0,8874.83,0.0...|    0|[3.83233030863620...|[0.97880008629612...|       0.0|\n",
      "|[31.0,5387.75,0.0...|    0|[3.24742811458119...|[0.96258058552664...|       0.0|\n",
      "|[31.0,7073.61,0.0...|    0|[3.79911450433881...|[0.97809976923405...|       0.0|\n",
      "|[31.0,11297.57,1....|    1|[0.79751152640735...|[0.68944192100551...|       0.0|\n",
      "|[31.0,11743.24,0....|    0|[7.95951793845681...|[0.99965080051155...|       0.0|\n",
      "|[31.0,12264.68,1....|    0|[3.77281170068563...|[0.97752920495855...|       0.0|\n",
      "|[32.0,6367.22,1.0...|    0|[3.20017220414578...|[0.96084075703562...|       0.0|\n",
      "|[32.0,8575.71,0.0...|    0|[4.52857300143358...|[0.98931923918898...|       0.0|\n",
      "|[32.0,13630.93,0....|    0|[2.65527248795398...|[0.93433521477806...|       0.0|\n",
      "|[33.0,4711.89,0.0...|    0|[7.15048703176813...|[0.99921613300884...|       0.0|\n",
      "|[33.0,5738.82,0.0...|    0|[5.41122451678732...|[0.99555369000330...|       0.0|\n",
      "|[33.0,7750.54,1.0...|    0|[4.79456321095382...|[0.99179329500352...|       0.0|\n",
      "|[33.0,12638.51,1....|    0|[4.15248449384766...|[0.98451815808214...|       0.0|\n",
      "|[33.0,13314.19,0....|    0|[3.36990907218523...|[0.96675076852634...|       0.0|\n",
      "|[34.0,5447.16,1.0...|    0|[3.75995719191832...|[0.97724510462861...|       0.0|\n",
      "|[34.0,6461.86,1.0...|    0|[4.80281076454080...|[0.99186015320798...|       0.0|\n",
      "|[34.0,7818.13,0.0...|    0|[4.73016790727597...|[0.99125221001613...|       0.0|\n",
      "|[34.0,9265.59,0.0...|    0|[4.83050636756087...|[0.99208073716831...|       0.0|\n",
      "+--------------------+-----+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_and_labels.predictions.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "churn_eval = BinaryClassificationEvaluator(rawPredictionCol='prediction',\n",
    "                                           labelCol='churn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc = churn_eval.evaluate(pred_and_labels.predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6866883116883117"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Common question - what is a good AUC value?](https://stats.stackexchange.com/questions/113326/what-is-a-good-auc-for-a-precision-recall-curve)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict on brand new unlabeled data\n",
    "\n",
    "We still need to evaluate the new_customers.csv file!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_lr_model = lr_churn.fit(final_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_customers = spark.read.csv('new_customers.csv',inferSchema=True,\n",
    "                              header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Names: string (nullable = true)\n",
      " |-- Age: double (nullable = true)\n",
      " |-- Total_Purchase: double (nullable = true)\n",
      " |-- Account_Manager: integer (nullable = true)\n",
      " |-- Years: double (nullable = true)\n",
      " |-- Num_Sites: double (nullable = true)\n",
      " |-- Onboard_date: timestamp (nullable = true)\n",
      " |-- Location: string (nullable = true)\n",
      " |-- Company: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "new_customers.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_new_customers = assembler.transform(new_customers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Names: string (nullable = true)\n",
      " |-- Age: double (nullable = true)\n",
      " |-- Total_Purchase: double (nullable = true)\n",
      " |-- Account_Manager: integer (nullable = true)\n",
      " |-- Years: double (nullable = true)\n",
      " |-- Num_Sites: double (nullable = true)\n",
      " |-- Onboard_date: timestamp (nullable = true)\n",
      " |-- Location: string (nullable = true)\n",
      " |-- Company: string (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_new_customers.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_results = final_lr_model.transform(test_new_customers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------+----------+\n",
      "|         Company|prediction|\n",
      "+----------------+----------+\n",
      "|        King Ltd|       0.0|\n",
      "|   Cannon-Benson|       1.0|\n",
      "|Barron-Robertson|       1.0|\n",
      "|   Sexton-Golden|       1.0|\n",
      "|        Wood LLC|       0.0|\n",
      "|   Parks-Robbins|       1.0|\n",
      "+----------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "final_results.select('Company','prediction').show()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}