Ridge_Lasso_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "\n",
    "# Numerical libraries\n",
    "import numpy as np   \n",
    "\n",
    "# Import Linear Regression machine learning library\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# to handle data in form of rows and columns \n",
    "import pandas as pd    \n",
    "\n",
    "# importing ploting libraries\n",
    "import matplotlib.pyplot as plt   \n",
    "\n",
    "#importing seaborn for statistical plots\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mpg_df = pd.read_csv(\"car-mpg.csv\")  \n",
    "mpg_df = mpg_df.drop('car_name', axis=1)\n",
    "mpg_df['origin'] = mpg_df['origin'].replace({1: 'america', 2: 'europe', 3: 'asia'})\n",
    "mpg_df = pd.get_dummies(mpg_df, columns=['origin'])\n",
    "mpg_df = mpg_df.replace('?', np.nan)\n",
    "mpg_df = mpg_df.apply(lambda x: x.fillna(x.median()),axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# separate independent and dependent variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copy all the predictor variables into X dataframe. Since 'mpg' is dependent variable drop it\n",
    "X = mpg_df.drop('mpg', axis=1)\n",
    "\n",
    "# Copy the 'mpg' column alone into the y dataframe. This is the dependent variable\n",
    "y = mpg_df[['mpg']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "# scale all the columns of the mpg_df. This will produce a numpy array\n",
    "X_scaled = preprocessing.scale(X)\n",
    "X_scaled = pd.DataFrame(X_scaled, columns=X.columns)  # ideally the training and test should be \n",
    "\n",
    "y_scaled = preprocessing.scale(y)\n",
    "y_scaled = pd.DataFrame(y_scaled, columns=y.columns)  # ideally the training and test should be "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_scaled, test_size=0.30, random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# fit a simple linear model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The coefficient for cyl is 0.3210223856916102\n",
      "The coefficient for disp is 0.3248343091848389\n",
      "The coefficient for hp is -0.2291695005943763\n",
      "The coefficient for wt is -0.7112101905072293\n",
      "The coefficient for acc is 0.014713682764191417\n",
      "The coefficient for yr is 0.3755811949510745\n",
      "The coefficient for car_type is 0.38147694842331037\n",
      "The coefficient for origin_america is -0.07472247547584179\n",
      "The coefficient for origin_asia is 0.04451525203567797\n",
      "The coefficient for origin_europe is 0.048348549539453875\n"
     ]
    }
   ],
   "source": [
    "regression_model = LinearRegression()\n",
    "regression_model.fit(X_train, y_train)\n",
    "\n",
    "for idx, col_name in enumerate(X_train.columns):\n",
    "    print(\"The coefficient for {} is {}\".format(col_name, regression_model.coef_[0][idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The intercept for our model is 0.01928411610363971\n"
     ]
    }
   ],
   "source": [
    "intercept = regression_model.intercept_[0]\n",
    "\n",
    "print(\"The intercept for our model is {}\".format(intercept))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a regularized RIDGE model and note the coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ridge model: [[ 0.31649043  0.31320707 -0.22876025 -0.70109447  0.01295851  0.37447352\n",
      "   0.37725608 -0.07423624  0.04441039  0.04784031]]\n"
     ]
    }
   ],
   "source": [
    "ridge = Ridge(alpha=.3)\n",
    "ridge.fit(X_train,y_train)\n",
    "print (\"Ridge model:\", (ridge.coef_))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a regularized LASSO model and note the coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso model: [-0.         -0.         -0.01690287 -0.51890013  0.          0.28138241\n",
      "  0.1278489  -0.01642647  0.          0.        ]\n"
     ]
    }
   ],
   "source": [
    "lasso = Lasso(alpha=0.1)\n",
    "lasso.fit(X_train,y_train)\n",
    "print (\"Lasso model:\", (lasso.coef_))\n",
    "\n",
    "# Observe, many of the coefficients have become 0 indicating drop of those dimensions from the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let us compare their scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8343770256960538\n",
      "0.8513421387780067\n"
     ]
    }
   ],
   "source": [
    "print(regression_model.score(X_train, y_train))\n",
    "print(regression_model.score(X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8343617931312616\n",
      "0.8518882171608506\n"
     ]
    }
   ],
   "source": [
    "print(ridge.score(X_train, y_train))\n",
    "print(ridge.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7938010766228453\n",
      "0.8375229615977083\n"
     ]
    }
   ],
   "source": [
    "print(lasso.score(X_train, y_train))\n",
    "print(lasso.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# More or less similar results but with less complex models.  Complexity is a function of variables and coefficients\n",
    "## Note - with Lasso, we get equally good result in test though not so in training.  Further, the number of dimensions is much less\n",
    "# in LASSO model than ridge or un-regularized model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let us generate polynomial models reflecting the non-linear interaction between some dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "poly = PolynomialFeatures(degree = 2, interaction_only=True)\n",
    "\n",
    "#poly = PolynomialFeatures(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(278, 56)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_poly = poly.fit_transform(X_scaled)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_poly, y, test_size=0.30, random_state=1)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit a simple non regularized linear model on poly features-"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-9.67853872e-13 -1.06354274e+13 -4.20438564e+00 -1.84192048e+00\n",
      " -2.72638006e+00 -1.26167775e+00  3.11178320e+00 -1.41608532e+13\n",
      " -1.75748808e+14 -1.00552702e+14 -1.67776805e+14 -2.24530835e+00\n",
      "  1.72983592e+00 -1.43166279e+00  5.23339190e+00 -2.40839721e+00\n",
      " -3.40741702e+13 -1.33084675e+13  1.62631150e+13  1.55231834e+13\n",
      "  3.94181666e-02 -1.20973679e-01 -1.74919750e+00  2.64875467e+00\n",
      " -1.96915695e+00 -1.01987477e+13 -8.40548787e+12 -8.02305890e+12\n",
      " -5.16931724e-01 -3.96352229e-01 -1.62429132e+00  1.23706751e+00\n",
      "  1.05966794e+13  8.73345073e+12  8.33610026e+12 -2.50000000e-01\n",
      "  3.00781250e-01 -5.60937500e+00 -1.49082566e+13 -1.22869174e+13\n",
      " -1.17278929e+13 -1.71875000e-01  3.98437500e+00  7.41223011e+12\n",
      "  6.10892753e+12  5.83098639e+12  8.59375000e-02 -2.52292482e+13\n",
      " -2.07931549e+13 -1.98471176e+13  1.63473760e+13  2.94730682e+13\n",
      "  2.81321163e+13  1.08552233e+14  5.18717282e+13 -8.66275359e+13]\n"
     ]
    }
   ],
   "source": [
    "regression_model.fit(X_train, y_train)\n",
    "print(regression_model.coef_[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ridge model: [[ 0.          3.73512981 -2.93500874 -2.13974194 -3.56547812 -1.28898893\n",
      "   3.01290805  2.04739082  0.0786974   0.21972225 -0.3302341  -1.46231096\n",
      "  -1.17221896  0.00856067  2.48054694 -1.67596093  0.99537516 -2.29024279\n",
      "   4.7699338  -2.08598898  0.34009408  0.35024058 -0.41761834  3.06970569\n",
      "  -2.21649433  1.86339518 -2.62934278  0.38596397  0.12088534 -0.53440382\n",
      "  -1.88265835 -0.7675926  -0.90146842  0.52416091  0.59678246 -0.26349448\n",
      "   0.5827378  -3.02842915 -0.36548074  0.5956112  -0.15941014  0.49168856\n",
      "   1.45652375 -0.43819158 -0.20964198  0.77665496  0.36489921 -0.4750838\n",
      "   0.3551047   0.23188557 -1.42941282  2.06831543 -0.34986402 -0.32320394\n",
      "   0.39054656  0.06283411]]\n"
     ]
    }
   ],
   "source": [
    "ridge = Ridge(alpha=.3)\n",
    "ridge.fit(X_train,y_train)\n",
    "print (\"Ridge model:\", (ridge.coef_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9143225702003364\n",
      "0.8613398053698547\n"
     ]
    }
   ],
   "source": [
    "print(ridge.score(X_train, y_train))\n",
    "print(ridge.score(X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso model: [ 0.          0.52263805 -0.5402102  -1.99423315 -4.55360385 -0.85285179\n",
      "  2.99044036  0.00711821 -0.          0.76073274 -0.         -0.\n",
      " -0.19736449  0.          2.04221833 -1.00014513  0.         -0.\n",
      "  4.28412669 -0.          0.          0.31442062 -0.          2.13894094\n",
      " -1.06760107  0.         -0.          0.          0.         -0.44991392\n",
      " -1.55885506 -0.         -0.68837902  0.          0.17455864 -0.34653644\n",
      "  0.3313704  -2.84931966  0.         -0.34340563  0.00815105  0.47019445\n",
      "  1.25759712 -0.69634581  0.          0.55528147  0.2948979  -0.67289549\n",
      "  0.06490671  0.         -1.19639935  1.06711702  0.         -0.88034391\n",
      "  0.         -0.        ]\n"
     ]
    }
   ],
   "source": [
    "lasso = Lasso(alpha=0.01)\n",
    "lasso.fit(X_train,y_train)\n",
    "print (\"Lasso model:\", (lasso.coef_))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9098286193898272\n",
      "0.8695296858772456\n"
     ]
    }
   ],
   "source": [
    "print(lasso.score(X_train, y_train))\n",
    "print(lasso.score(X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}