Feature Selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from scipy.stats import norm\n",
    "import scipy.stats as sps\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.feature_selection import RFE\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 28.2 s, sys: 7.45 s, total: 35.7 s\n",
      "Wall time: 37 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record</th>\n",
       "      <th>date</th>\n",
       "      <th>ssn_#days_since</th>\n",
       "      <th>ssn_lag0_count</th>\n",
       "      <th>ssn_lag1_count</th>\n",
       "      <th>ssn_lag3_count</th>\n",
       "      <th>ssn_lag7_count</th>\n",
       "      <th>ssn_lag14_count</th>\n",
       "      <th>ssn_lag30_count</th>\n",
       "      <th>ssn_lag1_lag3_avg</th>\n",
       "      <th>...</th>\n",
       "      <th>name_dob_30_day_cross_count</th>\n",
       "      <th>name_homephone_0_day_cross_count</th>\n",
       "      <th>name_homephone_1_day_cross_count</th>\n",
       "      <th>name_homephone_3_day_cross_count</th>\n",
       "      <th>name_homephone_7_day_cross_count</th>\n",
       "      <th>name_homephone_14_day_cross_count</th>\n",
       "      <th>name_homephone_30_day_cross_count</th>\n",
       "      <th>dow</th>\n",
       "      <th>fraud_label</th>\n",
       "      <th>dow_risk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.014499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.014499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.014499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.014499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.014499</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 521 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   record        date  ssn_#days_since  ssn_lag0_count  ssn_lag1_count  \\\n",
       "0       1  2016-01-01              0.0               1               1   \n",
       "1       2  2016-01-01              0.0               1               1   \n",
       "2       3  2016-01-01              0.0               1               1   \n",
       "3       4  2016-01-01              0.0               1               1   \n",
       "4       5  2016-01-01              0.0               1               1   \n",
       "\n",
       "   ssn_lag3_count  ssn_lag7_count  ssn_lag14_count  ssn_lag30_count  \\\n",
       "0               1               1                1                1   \n",
       "1               1               1                1                1   \n",
       "2               1               1                1                1   \n",
       "3               1               1                1                1   \n",
       "4               1               1                1                1   \n",
       "\n",
       "   ssn_lag1_lag3_avg  ...  name_dob_30_day_cross_count  \\\n",
       "0                3.0  ...                            1   \n",
       "1                3.0  ...                            1   \n",
       "2                3.0  ...                            1   \n",
       "3                3.0  ...                            1   \n",
       "4                3.0  ...                            1   \n",
       "\n",
       "   name_homephone_0_day_cross_count  name_homephone_1_day_cross_count  \\\n",
       "0                                 1                                 1   \n",
       "1                                 1                                 1   \n",
       "2                                 1                                 1   \n",
       "3                                 1                                 1   \n",
       "4                                 1                                 1   \n",
       "\n",
       "   name_homephone_3_day_cross_count  name_homephone_7_day_cross_count  \\\n",
       "0                                 1                                 1   \n",
       "1                                 1                                 1   \n",
       "2                                 1                                 1   \n",
       "3                                 1                                 1   \n",
       "4                                 1                                 1   \n",
       "\n",
       "   name_homephone_14_day_cross_count  name_homephone_30_day_cross_count  dow  \\\n",
       "0                                  1                                  1    4   \n",
       "1                                  1                                  1    4   \n",
       "2                                  1                                  1    4   \n",
       "3                                  1                                  1    4   \n",
       "4                                  1                                  1    4   \n",
       "\n",
       "   fraud_label  dow_risk  \n",
       "0            0  0.014499  \n",
       "1            1  0.014499  \n",
       "2            0  0.014499  \n",
       "3            0  0.014499  \n",
       "4            0  0.014499  \n",
       "\n",
       "[5 rows x 521 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "data = pd.read_csv('finalDF.csv', index_col=0)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000000, 521)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oot.shape: (166493, 522)\n",
      "tt.shape: (794996, 522)\n",
      "CPU times: user 24.5 s, sys: 36.6 s, total: 1min 1s\n",
      "Wall time: 1min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# remove the first 2 weeks and the last 2 months data in 2016\n",
    "# because data of first 2 weeks may not be robust and data of last 2 month is for validation\n",
    "\n",
    "df = data.loc[(data['date']>'2016-01-14')].copy()\n",
    "\n",
    "# Remove non-variable columns \n",
    "df.drop(columns=['date','fraud_label'],inplace=True)\n",
    "\n",
    "# Standarize variables first to speed up converging in wrapper (backward selection) \n",
    "scaler = StandardScaler()\n",
    "df_scaled = scaler.fit_transform(df)\n",
    "df_scaled = pd.DataFrame(df)\n",
    "df_scaled.columns = df.columns\n",
    "df_scaled['date'] = data.loc[(data['date']>'2016-01-14')]['date'].values\n",
    "df_scaled['fraud_label'] = data.loc[(data['date']>'2016-01-14')]['fraud_label'].values\n",
    "\n",
    "\n",
    "# Add a random variable ~ N(0,1)\n",
    "dist = norm(0,1)\n",
    "df_scaled['RANDOM'] = dist.rvs(size = df.shape[0])\n",
    "\n",
    "# Set the data from the last two months in 2016 as out of time (OOT) data\n",
    "oot = df_scaled.loc[df_scaled['date']>='2016-11-01']\n",
    "tt = df_scaled.loc[df_scaled['date']<'2016-11-01']\n",
    "\n",
    "print(f'oot.shape: {oot.shape}')\n",
    "print(f'tt.shape: {tt.shape}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 12s, sys: 7.82 s, total: 1min 20s\n",
      "Wall time: 1min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "goods = tt[tt['fraud_label']==0]\n",
    "bads = tt[tt['fraud_label']==1]\n",
    "cols = list(tt.columns)\n",
    "cols.remove('date')\n",
    "KSFDR = pd.DataFrame()\n",
    "\n",
    "for var in cols:\n",
    "    KSFDR.loc[var,'ks'] = sps.ks_2samp(goods[var],bads[var])[0]\n",
    "    KSFDR.loc[var,'ks_pvalue'] = sps.ks_2samp(goods[var],bads[var])[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ks</th>\n",
       "      <th>ks_pvalue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fraud_label</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>address_lag30_count</th>\n",
       "      <td>0.332725</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fulladdress_lag30_count</th>\n",
       "      <td>0.332032</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>address_#days_since</th>\n",
       "      <td>0.324627</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fulladdress_#days_since</th>\n",
       "      <td>0.323543</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn_dob_3_day_cross_count</th>\n",
       "      <td>0.000128</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn_dob_1_day_cross_count</th>\n",
       "      <td>0.000126</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn_dob_0_day_cross_count</th>\n",
       "      <td>0.000100</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn_name_1_day_cross_count</th>\n",
       "      <td>0.000085</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn_dob_7_day_cross_count</th>\n",
       "      <td>0.000074</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>521 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  ks  ks_pvalue\n",
       "fraud_label                 1.000000        0.0\n",
       "address_lag30_count         0.332725        0.0\n",
       "fulladdress_lag30_count     0.332032        0.0\n",
       "address_#days_since         0.324627        0.0\n",
       "fulladdress_#days_since     0.323543        0.0\n",
       "...                              ...        ...\n",
       "ssn_dob_3_day_cross_count   0.000128        1.0\n",
       "ssn_dob_1_day_cross_count   0.000126        1.0\n",
       "ssn_dob_0_day_cross_count   0.000100        1.0\n",
       "ssn_name_1_day_cross_count  0.000085        1.0\n",
       "ssn_dob_7_day_cross_count   0.000074        1.0\n",
       "\n",
       "[521 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KSFDR.sort_values('ks',ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FDR at 3%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17.6 s, sys: 3.28 s, total: 20.9 s\n",
      "Wall time: 21 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "topRows = int(round(len(tt)*0.03))\n",
    "numbads = bads.shape[0]\n",
    "for var in cols:\n",
    "    if var =='fraud_label':\n",
    "        temp = tt[[var]]\n",
    "    else:\n",
    "        temp = tt[[var,'fraud_label']]\n",
    "    temp0 = temp.sort_values(var)\n",
    "    temp1 = temp0.head(numbads)\n",
    "    temp2 = temp0.tail(numbads)\n",
    "    needed1 = temp1.loc[:,'fraud_label']\n",
    "    needed2 = temp2.loc[:,'fraud_label']\n",
    "    FDR1 = sum(needed1)/numbads\n",
    "    FDR2 = sum(needed2)/numbads\n",
    "    FDRate = max(FDR1,FDR2)\n",
    "    KSFDR.loc[var,'FDR at 3%'] = FDRate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ks</th>\n",
       "      <th>ks_pvalue</th>\n",
       "      <th>FDR at 3%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fraud_label</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fulladdress_lag30_count</th>\n",
       "      <td>0.332032</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.334146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>address_lag30_count</th>\n",
       "      <td>0.332725</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.332666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fulladdress_#days_since</th>\n",
       "      <td>0.323543</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.317778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fulladdress_lag14_count</th>\n",
       "      <td>0.321756</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.314644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dob_ssn_30_day_cross_count</th>\n",
       "      <td>0.064100</td>\n",
       "      <td>7.033715e-41</td>\n",
       "      <td>0.014365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RANDOM</th>\n",
       "      <td>0.008857</td>\n",
       "      <td>3.349735e-01</td>\n",
       "      <td>0.014017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dob_name_30_day_cross_count</th>\n",
       "      <td>0.064041</td>\n",
       "      <td>8.349334e-41</td>\n",
       "      <td>0.013930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dob_ssn_14_day_cross_count</th>\n",
       "      <td>0.046688</td>\n",
       "      <td>7.005442e-22</td>\n",
       "      <td>0.013669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dob_name_14_day_cross_count</th>\n",
       "      <td>0.046373</td>\n",
       "      <td>1.361718e-21</td>\n",
       "      <td>0.013582</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>521 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   ks     ks_pvalue  FDR at 3%\n",
       "fraud_label                  1.000000  0.000000e+00   1.000000\n",
       "fulladdress_lag30_count      0.332032  0.000000e+00   0.334146\n",
       "address_lag30_count          0.332725  0.000000e+00   0.332666\n",
       "fulladdress_#days_since      0.323543  0.000000e+00   0.317778\n",
       "fulladdress_lag14_count      0.321756  0.000000e+00   0.314644\n",
       "...                               ...           ...        ...\n",
       "dob_ssn_30_day_cross_count   0.064100  7.033715e-41   0.014365\n",
       "RANDOM                       0.008857  3.349735e-01   0.014017\n",
       "dob_name_30_day_cross_count  0.064041  8.349334e-41   0.013930\n",
       "dob_ssn_14_day_cross_count   0.046688  7.005442e-22   0.013669\n",
       "dob_name_14_day_cross_count  0.046373  1.361718e-21   0.013582\n",
       "\n",
       "[521 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KSFDR.sort_values('FDR at 3%',ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sort ks and FDR by average rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "KSFDR['rank_ks'] = KSFDR['ks'].rank(ascending=True)\n",
    "KSFDR['rank_FDR'] = KSFDR['FDR at 3%'].rank(ascending=True)\n",
    "KSFDR['avg_rank'] = (KSFDR['rank_ks']+KSFDR['rank_FDR'])/2\n",
    "KSFDR.sort_values('avg_rank',ascending=False,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ks</th>\n",
       "      <th>ks_pvalue</th>\n",
       "      <th>FDR at 3%</th>\n",
       "      <th>rank_ks</th>\n",
       "      <th>rank_FDR</th>\n",
       "      <th>avg_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fraud_label</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>521.0</td>\n",
       "      <td>521.0</td>\n",
       "      <td>521.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>address_lag30_count</th>\n",
       "      <td>0.332725</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.332666</td>\n",
       "      <td>520.0</td>\n",
       "      <td>519.0</td>\n",
       "      <td>519.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fulladdress_lag30_count</th>\n",
       "      <td>0.332032</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.334146</td>\n",
       "      <td>519.0</td>\n",
       "      <td>520.0</td>\n",
       "      <td>519.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fulladdress_#days_since</th>\n",
       "      <td>0.323543</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.317778</td>\n",
       "      <td>517.0</td>\n",
       "      <td>518.0</td>\n",
       "      <td>517.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>address_#days_since</th>\n",
       "      <td>0.324627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.308462</td>\n",
       "      <td>518.0</td>\n",
       "      <td>515.0</td>\n",
       "      <td>516.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn_dob_30_day_cross_count</th>\n",
       "      <td>0.000816</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.017064</td>\n",
       "      <td>15.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>16.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn_dob_3_day_cross_count</th>\n",
       "      <td>0.000128</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.018022</td>\n",
       "      <td>5.0</td>\n",
       "      <td>26.5</td>\n",
       "      <td>15.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name_dob_0_day_cross_count</th>\n",
       "      <td>0.000327</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.017761</td>\n",
       "      <td>9.0</td>\n",
       "      <td>21.5</td>\n",
       "      <td>15.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dob_ssn_0_day_cross_count</th>\n",
       "      <td>0.001426</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.015845</td>\n",
       "      <td>19.0</td>\n",
       "      <td>9.5</td>\n",
       "      <td>14.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dob_name_0_day_cross_count</th>\n",
       "      <td>0.001425</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.015845</td>\n",
       "      <td>18.0</td>\n",
       "      <td>9.5</td>\n",
       "      <td>13.75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>521 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  ks  ks_pvalue  FDR at 3%  rank_ks  rank_FDR  \\\n",
       "fraud_label                 1.000000        0.0   1.000000    521.0     521.0   \n",
       "address_lag30_count         0.332725        0.0   0.332666    520.0     519.0   \n",
       "fulladdress_lag30_count     0.332032        0.0   0.334146    519.0     520.0   \n",
       "fulladdress_#days_since     0.323543        0.0   0.317778    517.0     518.0   \n",
       "address_#days_since         0.324627        0.0   0.308462    518.0     515.0   \n",
       "...                              ...        ...        ...      ...       ...   \n",
       "ssn_dob_30_day_cross_count  0.000816        1.0   0.017064     15.0      17.0   \n",
       "ssn_dob_3_day_cross_count   0.000128        1.0   0.018022      5.0      26.5   \n",
       "name_dob_0_day_cross_count  0.000327        1.0   0.017761      9.0      21.5   \n",
       "dob_ssn_0_day_cross_count   0.001426        1.0   0.015845     19.0       9.5   \n",
       "dob_name_0_day_cross_count  0.001425        1.0   0.015845     18.0       9.5   \n",
       "\n",
       "                            avg_rank  \n",
       "fraud_label                   521.00  \n",
       "address_lag30_count           519.50  \n",
       "fulladdress_lag30_count       519.50  \n",
       "fulladdress_#days_since       517.50  \n",
       "address_#days_since           516.50  \n",
       "...                              ...  \n",
       "ssn_dob_30_day_cross_count     16.00  \n",
       "ssn_dob_3_day_cross_count      15.75  \n",
       "name_dob_0_day_cross_count     15.25  \n",
       "dob_ssn_0_day_cross_count      14.25  \n",
       "dob_name_0_day_cross_count     13.75  \n",
       "\n",
       "[521 rows x 6 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KSFDR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 91.1 ms, sys: 9.08 ms, total: 100 ms\n",
      "Wall time: 91.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# add variable number column\n",
    "for i in range(len(cols)):\n",
    "    KSFDR.loc[cols[i],'Variable number'] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variable number</th>\n",
       "      <th>Variable</th>\n",
       "      <th>ks</th>\n",
       "      <th>FDR at 3%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>519.0</td>\n",
       "      <td>fraud_label</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>73.0</td>\n",
       "      <td>fulladdress_lag30_count</td>\n",
       "      <td>0.332032</td>\n",
       "      <td>0.334146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18.0</td>\n",
       "      <td>address_lag30_count</td>\n",
       "      <td>0.332725</td>\n",
       "      <td>0.332666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>67.0</td>\n",
       "      <td>fulladdress_#days_since</td>\n",
       "      <td>0.323543</td>\n",
       "      <td>0.317778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>72.0</td>\n",
       "      <td>fulladdress_lag14_count</td>\n",
       "      <td>0.321756</td>\n",
       "      <td>0.314644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>450.0</td>\n",
       "      <td>dob_ssn_30_day_cross_count</td>\n",
       "      <td>0.064100</td>\n",
       "      <td>0.014365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>520.0</td>\n",
       "      <td>RANDOM</td>\n",
       "      <td>0.008857</td>\n",
       "      <td>0.014017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>468.0</td>\n",
       "      <td>dob_name_30_day_cross_count</td>\n",
       "      <td>0.064041</td>\n",
       "      <td>0.013930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>449.0</td>\n",
       "      <td>dob_ssn_14_day_cross_count</td>\n",
       "      <td>0.046688</td>\n",
       "      <td>0.013669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>467.0</td>\n",
       "      <td>dob_name_14_day_cross_count</td>\n",
       "      <td>0.046373</td>\n",
       "      <td>0.013582</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>521 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Variable number                     Variable        ks  FDR at 3%\n",
       "0              519.0                  fraud_label  1.000000   1.000000\n",
       "2               73.0      fulladdress_lag30_count  0.332032   0.334146\n",
       "1               18.0          address_lag30_count  0.332725   0.332666\n",
       "3               67.0      fulladdress_#days_since  0.323543   0.317778\n",
       "5               72.0      fulladdress_lag14_count  0.321756   0.314644\n",
       "..               ...                          ...       ...        ...\n",
       "401            450.0   dob_ssn_30_day_cross_count  0.064100   0.014365\n",
       "488            520.0                       RANDOM  0.008857   0.014017\n",
       "402            468.0  dob_name_30_day_cross_count  0.064041   0.013930\n",
       "432            449.0   dob_ssn_14_day_cross_count  0.046688   0.013669\n",
       "438            467.0  dob_name_14_day_cross_count  0.046373   0.013582\n",
       "\n",
       "[521 rows x 4 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KSFDR = KSFDR.reset_index().rename(columns={'index':'Variable'}).set_index('Variable number')\n",
    "sortbyKS = KSFDR.reset_index().sort_values('ks',ascending=False)\n",
    "sortbyKS = sortbyKS[['Variable number', 'Variable', 'ks', 'FDR at 3%']]\n",
    "sortbyFDR = KSFDR.reset_index().sort_values('FDR at 3%',ascending=False)\n",
    "sortbyFDR = sortbyFDR[['Variable number', 'Variable', 'ks', 'FDR at 3%']]\n",
    "sortbyFDR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variable number</th>\n",
       "      <th>Variable</th>\n",
       "      <th>ks</th>\n",
       "      <th>FDR at 3%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>519.0</td>\n",
       "      <td>fraud_label</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18.0</td>\n",
       "      <td>address_lag30_count</td>\n",
       "      <td>0.332725</td>\n",
       "      <td>0.332666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>73.0</td>\n",
       "      <td>fulladdress_lag30_count</td>\n",
       "      <td>0.332032</td>\n",
       "      <td>0.334146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12.0</td>\n",
       "      <td>address_#days_since</td>\n",
       "      <td>0.324627</td>\n",
       "      <td>0.308462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>67.0</td>\n",
       "      <td>fulladdress_#days_since</td>\n",
       "      <td>0.323543</td>\n",
       "      <td>0.317778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>405.0</td>\n",
       "      <td>ssn_dob_3_day_cross_count</td>\n",
       "      <td>0.000128</td>\n",
       "      <td>0.018022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>404.0</td>\n",
       "      <td>ssn_dob_1_day_cross_count</td>\n",
       "      <td>0.000126</td>\n",
       "      <td>0.018196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>403.0</td>\n",
       "      <td>ssn_dob_0_day_cross_count</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.018196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505</th>\n",
       "      <td>416.0</td>\n",
       "      <td>ssn_name_1_day_cross_count</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.018196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>406.0</td>\n",
       "      <td>ssn_dob_7_day_cross_count</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.018109</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>521 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Variable number                    Variable        ks  FDR at 3%\n",
       "0              519.0                 fraud_label  1.000000   1.000000\n",
       "1               18.0         address_lag30_count  0.332725   0.332666\n",
       "2               73.0     fulladdress_lag30_count  0.332032   0.334146\n",
       "4               12.0         address_#days_since  0.324627   0.308462\n",
       "3               67.0     fulladdress_#days_since  0.323543   0.317778\n",
       "..               ...                         ...       ...        ...\n",
       "517            405.0   ssn_dob_3_day_cross_count  0.000128   0.018022\n",
       "501            404.0   ssn_dob_1_day_cross_count  0.000126   0.018196\n",
       "503            403.0   ssn_dob_0_day_cross_count  0.000100   0.018196\n",
       "505            416.0  ssn_name_1_day_cross_count  0.000085   0.018196\n",
       "512            406.0   ssn_dob_7_day_cross_count  0.000074   0.018109\n",
       "\n",
       "[521 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sortbyKS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 191 ms, sys: 53.9 ms, total: 245 ms\n",
      "Wall time: 310 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Write your code to write to an excel file\n",
    "writer = pd.ExcelWriter('Variable statistics.xlsx')\n",
    "sortbyKS.to_excel(writer,sheet_name = 'sortbyKS', index = False)\n",
    "sortbyFDR.to_excel(writer,sheet_name='sortbyFDR', index=False)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fraud_label</th>\n",
       "      <th>address_lag30_count</th>\n",
       "      <th>fulladdress_lag30_count</th>\n",
       "      <th>fulladdress_#days_since</th>\n",
       "      <th>address_#days_since</th>\n",
       "      <th>fulladdress_lag14_count</th>\n",
       "      <th>fulladdress_lag7_count</th>\n",
       "      <th>address_lag14_count</th>\n",
       "      <th>address_lag7_count</th>\n",
       "      <th>address_ssn_30_day_cross_count</th>\n",
       "      <th>...</th>\n",
       "      <th>ssn_name_dob_lag7_count</th>\n",
       "      <th>name_lag14_count</th>\n",
       "      <th>address_lag0_count</th>\n",
       "      <th>fulladdress_lag0_count</th>\n",
       "      <th>fulladdress_lag1_lag7_avg</th>\n",
       "      <th>address_lag1_lag7_avg</th>\n",
       "      <th>homephone_lag3_count</th>\n",
       "      <th>address_name_0_day_cross_count</th>\n",
       "      <th>address_dob_0_day_cross_count</th>\n",
       "      <th>address_ssn_0_day_cross_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>38511</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38512</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38513</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38514</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38515</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       fraud_label  address_lag30_count  fulladdress_lag30_count  \\\n",
       "38511            0                    1                        1   \n",
       "38512            0                    1                        1   \n",
       "38513            0                    1                        1   \n",
       "38514            0                    1                        1   \n",
       "38515            0                    1                        1   \n",
       "\n",
       "       fulladdress_#days_since  address_#days_since  fulladdress_lag14_count  \\\n",
       "38511                     14.0                 14.0                        1   \n",
       "38512                     14.0                 14.0                        1   \n",
       "38513                     14.0                 14.0                        1   \n",
       "38514                     14.0                 14.0                        1   \n",
       "38515                     14.0                 14.0                        1   \n",
       "\n",
       "       fulladdress_lag7_count  address_lag14_count  address_lag7_count  \\\n",
       "38511                       1                    1                   1   \n",
       "38512                       1                    1                   1   \n",
       "38513                       1                    1                   1   \n",
       "38514                       1                    1                   1   \n",
       "38515                       1                    1                   1   \n",
       "\n",
       "       address_ssn_30_day_cross_count  ...  ssn_name_dob_lag7_count  \\\n",
       "38511                               1  ...                        1   \n",
       "38512                               1  ...                        1   \n",
       "38513                               1  ...                        1   \n",
       "38514                               1  ...                        1   \n",
       "38515                               1  ...                        1   \n",
       "\n",
       "       name_lag14_count  address_lag0_count  fulladdress_lag0_count  \\\n",
       "38511                 1                   1                       1   \n",
       "38512                 1                   1                       1   \n",
       "38513                 1                   1                       1   \n",
       "38514                 1                   1                       1   \n",
       "38515                 1                   1                       1   \n",
       "\n",
       "       fulladdress_lag1_lag7_avg  address_lag1_lag7_avg  homephone_lag3_count  \\\n",
       "38511                        7.0                    7.0                     1   \n",
       "38512                        7.0                    7.0                     2   \n",
       "38513                        7.0                    7.0                     1   \n",
       "38514                        7.0                    7.0                     1   \n",
       "38515                        7.0                    7.0                     1   \n",
       "\n",
       "       address_name_0_day_cross_count  address_dob_0_day_cross_count  \\\n",
       "38511                               1                              1   \n",
       "38512                               1                              1   \n",
       "38513                               1                              1   \n",
       "38514                               1                              1   \n",
       "38515                               1                              1   \n",
       "\n",
       "       address_ssn_0_day_cross_count  \n",
       "38511                              1  \n",
       "38512                              1  \n",
       "38513                              1  \n",
       "38514                              1  \n",
       "38515                              1  \n",
       "\n",
       "[5 rows x 81 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# choose top 81 (top80+fraud_label)\n",
    "var_list = KSFDR.reset_index().iloc[:81,:]['Variable']\n",
    "var1 = tt.loc[:,var_list]\n",
    "var1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['fraud_label', 'address_lag30_count', 'fulladdress_lag30_count',\n",
       "       'fulladdress_#days_since', 'address_#days_since',\n",
       "       'fulladdress_lag14_count', 'fulladdress_lag7_count',\n",
       "       'address_lag14_count', 'address_lag7_count',\n",
       "       'address_ssn_30_day_cross_count',\n",
       "       'address_name_30_day_cross_count', 'fulladdress_lag3_count',\n",
       "       'address_lag3_count', 'address_ssn_14_day_cross_count',\n",
       "       'address_name_14_day_cross_count',\n",
       "       'address_dob_30_day_cross_count', 'address_dob_14_day_cross_count',\n",
       "       'address_ssn_7_day_cross_count', 'address_name_7_day_cross_count',\n",
       "       'address_dob_7_day_cross_count', 'address_ssn_3_day_cross_count',\n",
       "       'address_dob_3_day_cross_count', 'address_name_3_day_cross_count',\n",
       "       'address_lag1_count', 'fulladdress_lag1_count',\n",
       "       'address_name_1_day_cross_count', 'address_dob_1_day_cross_count',\n",
       "       'address_ssn_1_day_cross_count', 'zip5_homephone_lag30_count',\n",
       "       'address_homephone_lag30_count',\n",
       "       'homephone_fulladdress_lag30_count', 'ssn_dob_lag30_count',\n",
       "       'lastname_dob_lag30_count', 'name_dob_lag30_count',\n",
       "       'firstname_dob_lag30_count', 'ssn_lag30_count',\n",
       "       'address_homephone_#days_since', 'ssn_name_dob_lag30_count',\n",
       "       'ssn_firstname_lag30_count', 'ssn_lastname_lag30_count',\n",
       "       'zip5_homephone_#days_since', 'homephone_fulladdress_#days_since',\n",
       "       'address_homephone_lag14_count', 'name_dob_#days_since',\n",
       "       'zip5_homephone_lag14_count', 'ssn_dob_#days_since',\n",
       "       'homephone_fulladdress_lag14_count', 'lastname_dob_#days_since',\n",
       "       'ssn_name_dob_#days_since', 'ssn_#days_since',\n",
       "       'name_dob_lag14_count', 'ssn_firstname_#days_since',\n",
       "       'ssn_dob_lag14_count', 'firstname_dob_#days_since',\n",
       "       'ssn_lastname_#days_since', 'lastname_dob_lag14_count',\n",
       "       'ssn_name_dob_lag14_count', 'ssn_lag14_count',\n",
       "       'ssn_firstname_lag14_count', 'firstname_dob_lag14_count',\n",
       "       'ssn_lastname_lag14_count', 'zip5_homephone_lag7_count',\n",
       "       'address_homephone_lag7_count', 'homephone_fulladdress_lag7_count',\n",
       "       'firstname_dob_lag7_count', 'lastname_dob_lag7_count',\n",
       "       'name_dob_lag7_count', 'ssn_dob_lag7_count', 'ssn_lag7_count',\n",
       "       'ssn_firstname_lag7_count', 'ssn_lastname_lag7_count',\n",
       "       'ssn_name_dob_lag7_count', 'name_lag14_count',\n",
       "       'address_lag0_count', 'fulladdress_lag0_count',\n",
       "       'fulladdress_lag1_lag7_avg', 'address_lag1_lag7_avg',\n",
       "       'homephone_lag3_count', 'address_name_0_day_cross_count',\n",
       "       'address_dob_0_day_cross_count', 'address_ssn_0_day_cross_count'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "var_list.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrapper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Backward selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 80 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 79 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 78 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 77 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 76 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 75 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 74 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 73 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 72 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 71 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 70 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 69 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 68 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 67 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 66 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 65 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 64 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 63 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 62 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 61 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 60 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 59 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 58 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 57 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 56 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 55 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 54 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 53 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 52 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 51 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 50 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 49 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 48 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 47 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 46 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 45 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 44 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 43 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 42 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 41 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 40 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 39 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 38 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 37 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 36 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 35 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 34 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 33 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 32 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 31 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 30 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 29 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 28 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 27 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 26 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 25 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 24 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 23 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 22 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 21 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 20 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 19 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 18 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 17 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 16 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 15 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 14 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 13 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 12 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 11 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 10 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 9 features.\n",
      "Fitting estimator with 8 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 7 features.\n",
      "Fitting estimator with 6 features.\n",
      "Fitting estimator with 5 features.\n",
      "Fitting estimator with 4 features.\n",
      "Fitting estimator with 3 features.\n",
      "Fitting estimator with 2 features.\n",
      "Fitting estimator with 80 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 79 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 78 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 77 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 76 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 75 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 74 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 73 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 72 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 71 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 70 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 69 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 68 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 67 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 66 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 65 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 64 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 63 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 62 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 61 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 60 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 59 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 58 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 57 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 56 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 55 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 54 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 53 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 52 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 51 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 50 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 49 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 48 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 47 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 46 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 45 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 44 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 43 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 42 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 41 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 40 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 39 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 38 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 37 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 36 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 35 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 34 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 33 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 32 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 31 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 30 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 29 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 28 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 27 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 26 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 25 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 24 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 23 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 22 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 21 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 20 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 19 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 18 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 17 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 16 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 15 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 14 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 13 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 12 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 11 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 10 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 9 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 8 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 7 features.\n",
      "Fitting estimator with 6 features.\n",
      "Fitting estimator with 5 features.\n",
      "Fitting estimator with 4 features.\n",
      "Fitting estimator with 3 features.\n",
      "Fitting estimator with 2 features.\n",
      "Fitting estimator with 80 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 79 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 78 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 77 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 76 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 75 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 74 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 73 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 72 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 71 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 70 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 69 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 68 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 67 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 66 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 65 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 64 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 63 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 62 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 61 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 60 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 59 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 58 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 57 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 56 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 55 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 54 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 53 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 52 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 51 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 50 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 49 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 48 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 47 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 46 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 52min 53s, sys: 4min 15s, total: 57min 9s\n",
      "Wall time: 15min 27s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# RFECV: feature ranking with recursive feature elimination and cross-validated selection \n",
    "# of the best number of features\n",
    "\n",
    "# First round wrapper\n",
    "X = var1.iloc[:,1:].values\n",
    "y = var1.iloc[:,0].values\n",
    "model = LogisticRegression(penalty = 'l2', class_weight = 'balanced')\n",
    "\n",
    "## parameters: 1)step: # of varibles to remove in each iteration\n",
    "rfecv1 = RFECV(estimator = model, step = 1, cv = 2, verbose = 3, n_jobs = 1, scoring = 'roc_auc')\n",
    "output1 = rfecv1.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of variables selected: 45\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of variables selected: {output1.get_support().sum()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEGCAYAAABcolNbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA0XklEQVR4nO3deZxcZZn3/8+3qnpJdzp7B7NBEmQRUAJEBHGQRRB1hEFFAWXcfiI+ouAoI47LD3XmkRFxGUdBRBY3EAUHBpFlMMCIC0kghCyAZCMbSYfs3emqrqrr+ePclVRXqqorSRVVnb7er1e9us459zl1VQfO1fdy7ltmhnPOOVcrsXoH4Jxzbv/micY551xNeaJxzjlXU55onHPO1ZQnGuecczWVqHcAjWjcuHE2derUeofhnHODxty5czeYWWexY55oipg6dSpz5sypdxjOOTdoSFpR6pg3nTnnnKspTzTOOedqyhONc865mvJE45xzrqY80TjnnKspTzTOOedqyhONc865mvLnaJwbZBav3cqSru0kYiImkYiLeCxGU0w0JWLEJHr7MmzrTbM9maY7maYvkyWVydKXNuIxOGBEKxNHDWPCyFbamhNs7E7xcneSjd0pWhJxXj1+OAeNbaMpvvvfoj2pNMs2dLO0q5u1W3Zw4Jg2jpw4ksmjhyFpt/K9fRm6tiXZsD1Jb1+WtuY4bc1xhjXHSWeMzTv62NSTYuuOPsa2t3Doq4bTObxl57XMjE09fXRtSzJlzDDamqt720qlsyzdsJ1p49ppScQHLJ/NGjv6MqTS0e80lc4ChO+VoLUphhkk01l6+zKkMlnGtjeTKPK7HCo80Tg3iPzsLyu46p6FZLK1X0cqERMHjm2jrTlOb19009yRyvByd6po+Y7WBAd3DiedzdKTytCbipLdtmR6jz97dFsTB3cOZ1tvmlWbeuhOZQCQYPq4do6YOJJDxw9nVFsT7S0JhrckaGtOkIiLREw7b+rpTJa+jNGXyZL/G8tmjUVrt/KXpS8zZ/kmdvRlGNPezHkzJ3Ph8Qdy0Nh2kukMTyzbyCPPdTFv5WY2dafYvKOPzT0pyv36JShc5qu9Oc4xB47muINGc8yBo2iKx9iRytDTlyGdydLZ0cKEkcOYOKqVlkScNZt3sPzlbpa/3APAW14zngkjhxX9PDPb+R3TGaOjNUEstnvCryf5wme7mzlzpvnMAK6RZLLGv/5uETc/vpzTDh/PFW89bOf+TNZIZ23nTTWdzdLWnKC9JU5HSxNtLXGaEzGa4zGa4jH6MlnWbe1l9eYdrN3cS09fhnHtzYxpb2bs8GZ6UhleWL+dJV3bWbK+m1QmS0siRmtTnNamOJNGtTJt3HCmd7YzYWQry1/uYdGarSxcs4UVL/fQnIgxrDlOW1Oc9pYEnR0tjBveTGdHC62JODv6MvSkoqQVj4nR7U2MHNbMyGEJ1m9L8txL23h+3TaWdHUzorWJKWOGMWV0G2OHN7NsQzcL12xl0ZqtrN68Y59/r4e/qoMTpo/liIkj+MPi9Ty0eB2ZrHHUpBEsWd/Njr4MzYkYM6aMYnxHC6PbmhnV1sTwlgQtiRjNiThNcWEQJY5Uhh2pNEi0NsVoDcefX7edOSs28exLW3dLQoXiMRX9Q+KYA0fxtqNexfiOVhau2cLCNVtZvHYrm3r6+pUb1hRnemc7rx4/nGnj2ulobaK1KUZLIk57c5yDxrYzvbOd1qao9pbOZHf+Xrcl01x0wkF79buUNNfMZhY95olmd55oXCPZnkxz2W1P8fCz6/nwSVP50juOIN5gf7HWQ655sDsZNRH2pDKks9Ff9ZmsYRiJWJRcm+LarVlv2rh2xrQ399u3bmsvv5q9kkeeW89rJ43klMPGc8L0sQxrHrhJrRJbe/tYtGYrwM4mxHgsxvqtvazd0suaLTvY3pvmwDFtTB3XztSx7XSn0ty/4CV+v2AtC1ZH5zYnYhz+qg6OnDiC8R2tNCei7xiTWLO5lyVd23lh/faSyViCKaPbGDmsiefXbSMZmv86O1p44l9OL9oEOhBPNHvIE41rFFt6+rjopr+ycM1WrnrnEVx04tR6h+TqaOXGHrpTaQ7uHF60/6xQMp2hN5Ulmc6QTGfZ2tvHsg3dvLA+SkRbdvRx2AEdHDFxBEdOHMnBne173ZdULtF4H41zDSqXZJ5du40ffeA43nLEAfUOydXZlDFte1S+JREPAxyadu47cuLIKkc1sKE7DMK5PWBm9PZlXrHPy08y133gWE8yblDzGo1zA3jupW1cedd8nnpxM9M72zl68iheO2kknR0tbNnRt/NlZjs7zFub4gxr2jWMt605TjzX7r3zh4gJJCFFu3Mjlr527yKeXbuN6y86ltMO9yTjBjdPNM6V0NuX4YezXuC6R5cwvCXBx0+ezpKubv60ZAO/fWp1v7ItuedX0pkBRxVVojke8yTj9hueaFxNzV2xif9+eg0AMeX+gt913AwMyJphFv3Mhjt17oYdjyl6KRpGumVHH5t7oucZcs9XQFQjiMUgEYuG8ibioiURo60lQVuoXcRi2nldM+s31LY3nYnOTUSvxWu2snRDN+86ZhJf+vsj+o1QWre1ly07+hg5rImRw5p2DhU1M1KZLL192TDcNRoNtaMvQzZrO5/liL539OWz4X3udwFwUBh15Nz+wBONq6lv3LeYeSs3M6w5jln03EfhyMlYaDrq9zPveNai50RyN+pRw5oY2dbMqGFNTBrV3O8BuaztenCtL5NlW2+ano099KQydCfTUbmdzVTRsw7tzQmGNcdpScTIZNMkwxPfw1sT/PQjx3PyobuvTnvAiFYOGNG6235JOztgRw5r2u24c0ORJxpXMxu7Uzz54iY+ddohfOaMQ+sdjnOuThpm1JmksyQ9J+kFSVcWOX6FpHnhtUBSRtIYSYfl7Z8naauky8M5V0lanXfs7a/4FxvCZj27nqzBW17j/QzODWUNUaORFAd+AJwBrAJmS7rHzBblypjZNcA1ofw7gc+Y2UZgIzAj7zqrgd/mXf47ZvatV+J7uP4efnYdB4xo4ahJI+odinOujhqlRnM88IKZLTWzFHA7cE6Z8hcAtxXZfzqwxMxW1CBGtwdS6SyPPb+B0w4fv1fTWTjn9h+NkmgmASvztleFfbuR1AacBdxZ5PD57J6ALpU0X9JNkkaXCkDSxZLmSJrT1dW1Z9G73fx12ctsT6Y53YfnOjfkNUqiKfYnb6mnEd4JPB6azXZdQGoGzgZ+nbf7OuBgoqa1tcC1pQIwsxvMbKaZzezs3H2UkdszDy9eT0sixkmvHlfvUJxzddYoiWYVMCVvezKwpkTZYrUWgLcBT5rZutwOM1tnZhkzywI/JmqiczVmZvzP4nW86dXjqjbrrXNu8GqURDMbOETStFAzOR+4p7CQpJHAm4G7i1xjt34bSRPyNs8FFlQtYlfS8+u2s2rTDk730WbOORpk1JmZpSVdCjwAxIGbzGyhpEvC8etD0XOBB82sO//80G9zBvDxgkt/U9IMoma45UWOuxr4n8VRpfL014yvcyTOuUbQEIkGwMzuA+4r2Hd9wfYtwC1Fzu0BxhbZf1FVg3QVeXjxOl47aWTRJ+edc0NPozSduf3Ehu1Jnlq52WszzrmdGqZG4wanlRt7+Nq9i+jty5DJGpt6+jCfDcA5l8drNG6f3PP0Gh5atI5tvWlS6SwdrQneO3MyR0702QCcc5Gq12gkXQbcDGwDbgSOAa40swer/Vmu/mYv38irxw/nvz55Ur1Dcc41qFrUaD5iZluBM4FO4MPA1TX4HFdnmawxd8UmXj+15IQLzjlXk0STe8r/7cDNZvY0xZ/8d4Pccy9tY1tvmtdPHVPvUJxzDawWiWaupAeJEs0DkjqAbA0+x9XZ7OXRLECeaJxz5dRi1NlHieYWW2pmPZLGEjWfuf3M7OUbedWIViaPHlbvUJxzDawWNRoDjgA+HbbbAX9ybz9jZsxevpHXTxvjywA458qqRaL5IXAi0dxjEI0++0ENPsfV0apNO1i3NekDAZxzA6pF09kbzOxYSU8BmNmmMFGm2488scz7Z5xzlalFjaYvLKlsAJI68cEA+505KzbS0ZrgsAM66h2Kc67B1SLR/AfwW2C8pH8D/gj83xp8jqujJ5ZtZOZBo4nFvH/GOVdeVZvOJMWAZcA/A6cTPT/zD2a2uJqf4+rr5e1JlnR18+7jJtc7FOfcIFDVRGNmWUnXmtmJwLPVvLZrHHNWbAK8f8Y5V5laNJ09KOnd8jGv+63ZyzbSnIjxuskj6x2Kc24QqMWos38ienYmLamXqPnMzMyn891PzF6xiRmTR9GSiNc7FOfcIFD1RGNmPgxpkEqmMyxZ383z67axtbePdMbIZI101pAgJhBi4eotXHzy9HqH65wbJGqxTMDJxfab2WPV/qzBbEcqw8pNPcRjojkeIxEX8ZgQCjd18dKWXhas3sIzq7ewYM0WNvf0kckaWTPMoCkuWpviDGuO05KIEctrrcya0duXpbcvQzKdJZ3N0t6cYHhLgvaWBC2JGFmLkkgma7y0pZdlG7pJZ23A2CU49XBfQdM5V5laNJ1dkfe+FTgemAucVu4kSWcB3wPiwI1mdnXB8SuA94fNBPAaoNPMNkpaTjQDQQZIm9nMcM4Y4FfAVGA58F4z27QP322vpTNZfvfMWuYs38RTKzexeO02MhXc1AGGtyQ4YuIIjpo0knhIQrGY6Mtk2ZHKsKMvE61wya7rCdHRmqCzo4VhTXHiMdGdTLM9mWZzT4pkOks8FiW3mMRBY9s488gDOOxVIzj8VR2MbW8mEYsRj4t4SGBZi5JcPCbamn1xVudcZWRW2c1urz9AmgJ808wuKFMmDjwPnAGsAmYDF5jZohLl3wl8xsxOC9vLgZlmtqGg3DeBjWZ2taQrgdFm9vmBYp45c6bNmTOnou9XqVnPrufDt8xmeEuCo6eMZMaUURwaHnbsyxh9mWxUmzDDADMY1dbEayeNZOrYdn9exTnX0CTNzf2RX+iV+LN0FXDUAGWOB14ws6UAkm4HzgGKJhqiedRuq+CzzwFOCe9vBR4BBkw0tfD8um0A/PHzpzKqzWfkcc4NHbXoo/k+7GzDiREtGfD0AKdNAlbmba8C3lDi+m3AWcClebuNaFi1AT8ysxvC/gPMbC2Ama2VVLJjQdLFwMUABx544ADh7rmlXd2MG97sScY5N+TUokaT3+aUBm4zs8cHOKdYu1CpNr13Ao+b2ca8fSeZ2ZqQSB6S9OyeDj4IyekGiJrO9uTcSizdsJ3p44ZX+7LOOdfwavHA5igzuzW8fmFmj0u6bIBzVgFT8rYnA2tKlD2fgmYzM1sTfq4nmmft+HBonaQJAOHn+j37KtWzpKub6Z3t9fp455yrm1okmg8W2fehAc6ZDRwiaVpYUuB84J7CQpJGAm8G7s7b1x6Wi0ZSO3AmsCAcvicvng/mn/dK2tyTYmN3yhONc25IqlrTmaQLgAuBaZLyk0QH8HK5c80sLelS4AGi4c03mdlCSZeE49eHoucCD5pZd97pBwC/DTPeJIBfmtn94djVwB2SPgq8CJy3L99xby3pisL1pjPn3FBUcaKR1F5wgy/0J2AtMA64Nm//NmD+QNc3s/uA+wr2XV+wfQtwS8G+pcDRJa75MtEs0nW1tGs7gNdonHND0oCJRtIbgRuB4cCBko4GPm5m/ye/nJmtAFYQLePs8izd0E0iJqaMaat3KM4594qrpI/mO8BbCc1fZvY0UHSaGQBJJ0iaLWm7pJSkjKSt1Ql3cFratZ0Dx7bRFK9Fl5hzzjW2iu58ZrayYFemTPH/JHqg8m/AMOD/A76/V9HtJ5Z2dXNwp/fPOOeGpkoSzcrQfGaSmiV9Dii7YqaZvQDEzSxjZjcDp1Yh1kEpncmy/GUf2uycG7oqGQxwCdFkl5OInnd5EPhkmfI9YYjyvDDX2Fqi9WmGpFWbdtCXMQ72EWfOuSGqbKIJk11+18zeX65cgYuIakqXAp8hehDz3Xsd4SC3dIOPOHPODW1lE42ZZSR1Smo2s1QlFzSzFZKGARPM7KtViXIQW5p7hsb7aJxzQ1QlTWfLgcfDQ5g7n6Mxs28XKxym8P8W0Ez08OYM4GtmdvY+RzsILenqZlRbE2PafTJN59zQVEmiWRNeMaKn/AdyFdFcY48AmNk8SVP3LrzBb2nXdqaP82Yz59zQNWCiyTV/hfnEzMy2D3BK2sy2SL5QF0QPa55yaGe9w3DOuboZcHizpKMkPUU0UeVCSXMlHVnmlAWSLgTikg4J69P8qUrxDipbe/vo2pb0/hnn3JBWyXM0NwD/ZGYHmdlBwGeBH5cp/yngSCAJ/BLYAly+j3EOSrsGAnjTmXNu6Kqkj6bdzGblNszskTAdfz+SfmZmFwEfM7MvAl+sYpyDUm4yzYM90TjnhrBKajRLJX1Z0tTw+hKwrEi54yQdBHxE0mhJY/Jf1Q17cFja1U08Jg4c44nGOTd0VVKj+QjwVeCusP0Y8OEi5a4H7gemA3Ppvzyzhf1DytIN25kyehjNCZ9M0zk3dFUy6mwT8OkKyv0H8B+SrjOzT1QjuMFuaVe3DwRwzg15lYw6e0jSqLzt0ZIeKFXek0wkkzWWbej2/hnn3JBXSZvOODPbnNsINZzxNYtoP7Fuay/JdJap/rCmc26IqyTRZCUdmNsIHf5Wu5D2Dz2pNADDWypeLds55/ZLldwFvwj8UdKjYftk4OLahbR/SKazALQk4nWOxDnn6mvAGo2Z3Q8cC/wKuAM4zsx266ORtE3S1lKvgT5H0lmSnpP0gqQrixy/QtK88FoQlogeI2mKpFmSFktaKOmyvHOukrQ677y3DxRHtexKND7izDk3tFUyGOAkYIeZ3QuMBP4lNJ/1Y2YdZjYC+C5wJdFCaZOBzwP/OsBnxIEfAG8DjgAukHREwfWvMbMZZjYD+ALwqJltBNLAZ83sNcAJwCcLzv1O7jwzu2+g71stKU80zjkHVNZHcx3RqplHA1cAK4Cflin/VjP7oZltM7OtZnYdAy98djzwgpktDeve3A6cU6b8BcBtAGa21syeDO+3ES0zPamC71VTuUTjz9A454a6Su6CaTMzohv/f5jZ9yi/XEBG0vslxSXFJL0fyAzwGZOAlXnbqyiRLCS1AWcBdxY5NhU4Bvhr3u5LJc2XdJOk0aUCkHSxpDmS5nR1dQ0Q7sC8j8Y55yKVJJptkr4AfAD4XWjmaipT/kLgvcC68Dov7Cun2JoCpUa2vRN4PDSb7bqANJwo+VxuZrk+oeuAg4EZwFrg2lIBmNkNZjbTzGZ2du77tP5eo3HOuUglo87eR5QoPmpmL4WhzteUKmxmyynf7FXMKmBK3vZkosXWijmf0GyWI6mJKMn8wsxyU+VgZuvyyvwYuHcP49pryXRUifM+GufcUFfJqLOXzOzbZva/YftFMyvZRyPpUEkPS1oQtl8XJuIsZzZwiKRpkpqJksk9Ra49EngzcHfePgE/ARYXLi8taULe5rlEa+q8IrxG45xzkVrcBX9MNCqsD8DM5hMljpLMLA1cCjxA1Jl/h5ktlHSJpEvyip4LPGhm3Xn7TgIuAk4rMoz5m5KekTQfOBX4TBW+X0V8eLNzzkVq8dh6m5k9UbCUc3qgk8LQ4/sK9l1fsH0LcEvBvj9SvI+HsD5OXXiNxjnnIrW4C26QdDChM1/Se4g64oeUXB+NJxrn3FA3YI0mPLB5FXBQKC/AzKzU+jKfJFr++XBJq4kWSXt/VaIdRHbWaOKeaJxzQ1slTWc/IerbmMvAz8NAlITeEpZ7jpnZNknT9iXIwSiZztKciFHQhOicc0NOJYlmi5n9fg+ueSdwbEGH/W+A4/YoskEumc76QADnnKOyRDNL0jVESzkncztz077kSDocOBIYKeldeYdGAK1ViHVQSWU80TjnHFSWaN4Qfs7M22fAaQXlDgP+HhhF9PR+zjbgY3sZ36CV7Mv69DPOOUcFicbMTq3kQmZ2N3C3pBPN7M/7HNkgl8pkfcSZc85R2TIBIyV9OzfhpKRrwxP6pVwiaVTe+aMl3VSNYAeTZF/Gm86cc47KnqO5iaj5673htRW4uUz515nZ5tyGmW0imlF5SPEajXPORSrpoznYzPLXk/mqpHllysckjQ4JBkljKvyc/UrUR+OJxjnnKkkAOyS9KUz1snPFzTLlrwX+JOk3RIMG3gv82z5HOsikMllamzzROOdcJYnmE8CtoV9GwEbgQ6UKm9lPJc0hGpUm4F1mtqgKsQ4qyXSGkcPKLdvjnHNDQyWjzuYBR0saEba3lj8DgDFAt5ndLKlT0jQzW7ZvoQ4uqXTWp59xzjnKJBpJHzCzn0v6p4L9ABSu/ZJ3/P8neubmMKJBA03Az4mm8x8yclPQOOfcUFeuRtMefnYUOVZqmWWI1ow5BngSwMzWSCp2jf1ayqegcc45oEyiMbMfhbf/Y2aP5x8LAwJKSZmZScotE9Bepux+K+U1GuecAyp7jub7Fe7LuUPSj4BRkj4G/A/RqptDSjSppk9B45xz5fpoTgTeCHQW9NOMAEreQc3sW5LOIHqw8zDgK2b2UJXiHTS8RuOcc5FyfTTNwPBQJr+PZSvwnnIXDYllyCWXnGzWfPZm55wLyvXRPAo8KukWM1sx0IUkbaP4IIHcipwjBjj/LOB7RLWlG83s6oLjV7Brpc4E8Bqg08w2ljo3zErwK2AqsBx4b27GglpKZcLqmp5onHOuoj6aHknXSLpP0h9yr8JCZtZhZiOKvDoqSDJx4AfA24AjgAskHVFw/WvMbIaZzQC+ADwakky5c68EHjazQ4CHw3bNJcMyzl6jcc65yhLNL4BngWnAV4lqBrPLnSDpTZI+HN6Pq2Ap5+OBF8xsqZmlgNuBc8qUvwC4rYJzzwFuDe9vBf5hgDiqIuWJxjnndqrkTjjWzH4C9JnZo2b2EeCEUoXDA5ufJ6p1QNTX8/MBPmMSsDJve1XYV+z6bcBZREtGD3TuAWa2FiD8HD9AHFWRTGcAfNSZc85RWaLpCz/XSnqHpGOAyWXKnwucDXRD9MAmxR/6zKci+0o9FPpO4HEz27gX55YOQLo4t+ZOV1fXnp7eT65G4300zjlXWaL51zCh5meBzwE3Ap8pUz5lZka42Vf4wOYqYEre9mRgTYmy57Or2Wygc9dJmhDimACsLxWAmd1gZjPNbGZnZ2cFIZeW9ETjnHM7DXgnNLN7zWyLmS0ws1PN7Dgzu6fMKXvzwOZs4BBJ0yQ1EyWT3T4jJLw3A3dXeO49wAfD+w8WnFcz3kfjnHO7lHtg8/uUaYIys0+X2L/HD2yaWVrSpcADREOUbzKzhZIuCcevD0XPBR40s+6Bzg2HryZKfB8FXgTOKxdHtfjwZuec26XcA5tzws+TiIYN/ypsnwfMLXVSaCr7g5k9JOkw4DBJTWbWV+ocADO7D7ivYN/1Bdu3ALdUcm7Y/zJwernPrYVkX65G44MBnHOu3AObtwJI+hBwai5RSLoeeLDMNR8D/k7SaKJmsznA+9j1sOV+L5WJRp15jcY55yobDDCR/qPGhod9pcjMeoB3Ad83s3OJakRDxq4ajSca55yrZCnnq4GnJM0K228GripTXmFCzvcDH92Dz9lveB+Nc87tUslSzjdL+j3whrDrSjN7qcwplxE9rPnb0KE/HZhVpvx+x2s0zjm3S7lRZ4eb2bOSjg27ck/fT5Q00cyeLHaemT1G1E+T214KFB2htr9Keo3GOed2Klej+SzwMeDaIscMOK0mEe0Hkn0+BY1zzuWUG3X2sfDz1FcunP1Dro/Gm86cc65809m7yp1oZncVlP93M/u8pPPM7NfVCnAwyvXRNMc90TjnXLmms3eWOWbAXQX73i7pS0QDAYZ0okllsjTFRSxWbL5P55wbWso1nX14D691P7ABaJe0lbCyJhWusLk/SaWzXptxzrmgoudbJL0DOBJoze0zs6/llzGzK4ArJN1tZuUWLdvvJdMZWpp8IIBzzkEFiSZMOdMGnEq0RMB7gCdKlTezcyQdALw+7Pqrme3bAi+DjNdonHNul0ruhm80s38ENpnZV4ET6b/+Sz+SziNKROcB7wWekPSeagQ7WCTTWVqaPNE45xxU1nS2I/zskTQReBmYVqb8l4DXm9l6AEmdRJNr/mZfAh1MvEbjnHO7VJJo7pU0CrgGeJKog7/cQmaxXJIJXqaymtN+w2s0zjm3SyVznX09vL1T0r1Aq5ltKXPK/ZIeYNdyy++jyFox+zOv0Tjn3C6VDAZ4mmjRs1+Z2RIgWa68mV0RHvZ8E9HQ5hvM7LfVCHawSKYzPv2Mc84FlTSdnU1UK7lDUpYo6dxhZi+WOiHMGlD4QOeQkUpnaWsbUisjOOdcSQO275jZCjP7ppkdB1wIvA5YVvPIBrFkOuvznDnnXFDpA5tTiYYqvw/IAP9cw5gGvVQ660sEOOdcMODdUNJfiZrBYsB5Zna8mRVbOgBJcUk/35tAJJ0l6TlJL0i6skSZUyTNk7RQ0qNh32FhX+61VdLl4dhVklbnHXv73sS2p5KeaJxzbqdKajQfNLNnK7mYmWUkdUpqNrNUpUFIigM/AM4AVgGzJd1jZovyyowCfgicZWYvShofPvM5YEbedVYD+YMPvmNm36o0lmqIms58MIBzzkFlw5srSjJ5lgOPS7oH6M67zrfLnHM88EJYjRNJtwPnAIvyylwI3JUbhFDwrE7O6cASM1uxhzFXVSqd8T4a55wLanE3XAPcG67dkfcqZxK7loqGqFYzqaDMocBoSY9ImivpH4tc53x2Pb+Tc6mk+ZJukjS6VACSLpY0R9Kcrq59m5rNBwM459wuVR+DG+ZDQ1K7mXUPVD4otnCLFWwngOOIai3DgD9L+ouZPR8+r5loKPYX8s65Dvh6uNbXiZal/kiJuG8AbgCYOXNm4WdXzMxIZbyPxjnncioZDHCepI7w/kuS7pJ0bJnyJ0paBCwO20dL+uEAH7OK/hN1TiaqGRWWud/Mus1sA/AYcHTe8bcBT5rZutwOM1tnZhkzyxJNm3P8AHHss76MYebLODvnXE4ld8Mvm9k2SW8C3grcSlRTKOW7odzLAGb2NHDyAJ8xGzhE0rRQMzkfuKegzN3A30lKSGoD3kBIZsEFFDSbSZqQt3kusGCAOPZZKhOWcfZE45xzQGVNZ5nw8x3AdWZ2t6Sryp1gZiulfq1hmVJlQ/m0pEuBB4A4cJOZLZR0STh+vZktlnQ/MB/IAjea2QKAkHjOAD5ecOlvSppB1HS2vMjxqkv2RV/VR50551ykkkSzWtKPgLcA/y6phfI1oZWS3ghYqJ18mv41j6LM7D4KJt80s+sLtq8hmkW68NweYGyR/RcN9LnV5jUa55zrr5K74XuJahpnmdlmYAxwRZnylwCfJBo1tproGZdP7lOUg0iyL0o03kfjnHORSmo0E4DfmVlS0ilEc539tFTh0FH//qpENwh5jcY55/qr5G54J5CR9GrgJ0Sra/6yVGFJ0yX9t6QuSesl3S1pepXibXi5Go2vR+Occ5FK7oZZM0sD7wK+a2afIarllPJL4I5QZiLwa3Z/iHK/lcqEwQBNPhjAOeegskTTJ+kC4B+JnvgHaCpTXmb2MzNLh9fP2f3hy/1WMu01Guecy1fJ3fDDwInAv5nZMknTgN1maJY0RtIYYJakKyVNlXSQpH8GflfdsBtXLtG0NHmicc45qGxSzUWSPgccKuko4Dkzu7pI0blENZfcAzT5z6zkpoDZ76W8RuOcc/0MmGjCSLNbiR54FDBF0gfN7LH8cmY2rQbxDTq5Gk2r12iccw6obHjztcCZYd0XJB1K1Ll/XLHCYU2YdwBT868/wDIB+41dNRofDOCcc1BZomnKJRkAM3teUrnBAP8N9ALPEE0VM6Qk07lRZ16jcc45qCzRzJX0E+BnYfv9RP0xpUw2s9ftc2SDlPfROOdcf5XcDS8BFhLNWXYZ0aqXl5Qp/3tJZ1YhtkHJR50551x/ZWs0kmLAXDM7Cqi0j+UvwG/DuX1EAwjMzEbsU6SDhNdonHOuv7J3w7Bg2NOSDtyDa15L9NxNm5mNMLOOoZJkIOqjicdEwhONc84BlU+quVDSE8DOpZnN7OwS5f8GLDCzITMbQL5UOuu1Geecy1NJovnqHl5zLfCIpN8DydzOoTS82Wduds65XUommjBb8wFm9mjB/pOJ1pkpZVl4NYfXkJJMZ30tGuecy1OuRvNd4F+K7O8Jx95Z7CQz29Ma0H7FazTOOddfuUQz1czmF+40szmSppY6SdIsiszWbGan7VWEg4zXaJxzrr9yiaa1zLFhZY59ruAa7wbSexLUYJZMZ2lO+PQzzjmXU+5P79mSPla4U9JHKTMzgJnNzXs9bmb/BLxhoEAknSXpOUkvSLqyRJlTJM2TtFDSo3n7l0t6Jhybk7d/jKSHJP0t/Bw9UBz7KpnOeI3GOefylKvRXE704GX+lDMziTr4zy11UliTJidGNPnmq8oFESbi/AFwBrCKKMndY2aL8sqMAn4InGVmL0oaX3CZU81sQ8G+K4GHzezqkLyuBD5fLpZ95X00zjnXX8lEY2brgDdKOhU4Kuz+nZn9YYBr5q9LkyYagfbRAc45HnjBzJYCSLodOIdoupucC4G7zOzFEN/6Aa5JuMYp4f2twCPUONEk01k6WisZNe6cc0NDJQufzQJmVXrBvVyXZhKwMm97Fbs3tx0KNEl6BOgAvmdmP819LPCgJAN+ZGY3hP0HmNnaENfaIrWgnSRdDFwMcOCBezIRQn8pHwzgnHP91ORPb0lvZPf1aH5a8oRdq3LmKxy5liBqhjudaDDCnyX9xcyeB04yszUhkTwk6dnChdkGEpLTDQAzZ87c61kNoj4aHwzgnHM5VU80kn4GHAzMAzJhtwHlEs0qYEre9mRgTZEyG8ysG+iW9BhwNPC8ma2BqDlN0m+JmuIeA9ZJmhBqMxOASprb9kkq4300zjmXrxY1mpnAEXs419ls4BBJ04hmHTifqE8m393Af0pKEA1IeAPwHUntQMzMtoX3ZwJfC+fcA3wQuDr8vHsvv1PFfK4z55zrrxaJZgHRKLO1lZ5gZmlJlwIPAHHgJjNbKOmScPx6M1ss6X5gPtHKnTea2QJJ04lGx0H0fX5pZveHS18N3BGGZL8InFedr1haMp31tWiccy5PLRLNOGBRmO05f1LNUrM9547fB9xXsO/6gu1rgGsK9i0lakIrds2Xifp0XjFeo3HOuf5qkWiuqsE1Bw2v0TjnXH9VTzSFsz0PJelMlkzWaI77qDPnnMvxP72rKJWJlnH2Go1zzu3id8QqSqWjRON9NM45t0vV7oiSHg4//71a1xxskmmv0TjnXKFq9tFMkPRm4OwwV1m/p/3N7MkqflZD8hqNc87trpqJ5itEsyNPBr5dcMyA/X7hs2Q6mgihpckHAzjnXE7VEo2Z/Qb4jaQvm9nXq3XdwSTpNRrnnNtNLYY3f13S2cDJYdcjZnZvtT+nEeWaznz2Zuec26Xqd0RJ3wAuI1pLZhFwWdi330t6onHOud3UYmaAdwAzzCwLIOlW4CngCzX4rIayczCAJxrnnNupVnfEUXnvR9boMxrOrhqNDwZwzrmcWtRovgE8JWkW0RDnkxkCtRnwGo1zzhVTi8EAt4Xlll9PlGg+b2YvVftzGtHO4c2eaJxzbqeaLOVsZmuJFh0bUrxG45xzu/M7YhX5qDPnnNud3xGryGs0zjm3O78jVtGuPhofdeacczmeaKooV6NpimuAks45N3Q0TKKRdJak5yS9IOnKEmVOkTRP0kJJj4Z9UyTNkrQ47L8sr/xVklaHc+ZJenstv0Myk6UlEUPyROOcczk1GXW2pyTFgR8AZwCrgNmS7jGzRXllRgE/BM4ysxcljQ+H0sBnzexJSR3AXEkP5Z37HTP71ivxPZJ9We+fcc65Ao1yVzweeMHMlppZCrgdOKegzIXAXWb2IoCZrQ8/1+bWujGzbcBiYNIrFnmeVKjROOec26VR7oqTgJV526vYPVkcCoyW9IikuZL+sfAikqYCxwB/zdt9qaT5km6SNLpUAJIuljRH0pyurq69+hLJvqwPBHDOuQKNkmiKdWpYwXYCOI5o0s63Al+WdOjOC0jDgTuBy81sa9h9HXAwMANYC1xbKgAzu8HMZprZzM7Ozr36EqmMN50551yhhuijIarBTMnbngysKVJmg5l1A92SHgOOBp6X1ESUZH5hZnflTjCzdbn3kn4M1HRdnGRfxpvOnHOuQKPcFWcDh0iaJqkZOJ/dp7C5G/g7SQlJbcAbgMWKhnj9BFhsZv2WkJY0IW/zXGBBzb4BXqNxzrliGqJGY2ZpSZcCDwBx4CYzWyjpknD8ejNbLOl+YD6QBW40swWS3gRcBDwjaV645L+Y2X3ANyXNIGqGWw58vJbfI+qj8UTjnHP5GiLRAITEcF/BvusLtq8BrinY90eK9/FgZhdVOcyyUpksrU2eaJxzLp/fFasomc74qDPnnCvgiaaKUukszXH/lTrnXD6/K1ZRKp2lxZvOnHOuH78rVlHSazTOObcbvytWUSrtw5udc66Q3xWrKJn2KWicc66QJ5oq8hqNc87tzu+KVWJmnHnkAbxmQke9Q3HOuYbSMA9sDnaS+M8Lj613GM4513C8RuOcc66mPNE455yrKU80zjnnasoTjXPOuZryROOcc66mPNE455yrKU80zjnnasoTjXPOuZqSmdU7hoYjqQtYUWHxccCGGoaztxo1Lmjc2Bo1LvDY9kajxgWNG9u+xHWQmXUWO+CJZh9JmmNmM+sdR6FGjQsaN7ZGjQs8tr3RqHFB48ZWq7i86cw551xNeaJxzjlXU55o9t0N9Q6ghEaNCxo3tkaNCzy2vdGocUHjxlaTuLyPxjnnXE15jcY551xNeaJxzjlXU55o9pKksyQ9J+kFSVfWOZabJK2XtCBv3xhJD0n6W/g5ug5xTZE0S9JiSQslXdZAsbVKekLS0yG2rzZKbCGOuKSnJN3bYHEtl/SMpHmS5jRKbJJGSfqNpGfDf28nNkhch4XfVe61VdLlDRLbZ8J/+wsk3Rb+n6hJXJ5o9oKkOPAD4G3AEcAFko6oY0i3AGcV7LsSeNjMDgEeDtuvtDTwWTN7DXAC8Mnwe2qE2JLAaWZ2NDADOEvSCQ0SG8BlwOK87UaJC+BUM5uR97xFI8T2PeB+MzscOJrod1f3uMzsufC7mgEcB/QAv613bJImAZ8GZprZUUAcOL9mcZmZv/bwBZwIPJC3/QXgC3WOaSqwIG/7OWBCeD8BeK4Bfm93A2c0WmxAG/Ak8IZGiA2YHP4nPw24t5H+PYHlwLiCfXWNDRgBLCMMbmqUuIrEeSbweCPEBkwCVgJjgARwb4ivJnF5jWbv5P6RclaFfY3kADNbCxB+jq9nMJKmAscAf6VBYgvNU/OA9cBDZtYosX0X+Gcgm7evEeICMOBBSXMlXdwgsU0HuoCbQ3PjjZLaGyCuQucDt4X3dY3NzFYD3wJeBNYCW8zswVrF5Ylm76jIPh8nXoKk4cCdwOVmtrXe8eSYWcaiJo3JwPGSjqpzSEj6e2C9mc2tdywlnGRmxxI1G39S0sn1DojoL/JjgevM7Bigm/o2Le5GUjNwNvDrescCEPpezgGmAROBdkkfqNXneaLZO6uAKXnbk4E1dYqllHWSJgCEn+vrEYSkJqIk8wszu6uRYssxs83AI0T9XPWO7STgbEnLgduB0yT9vAHiAsDM1oSf64n6Go5vgNhWAatCjRTgN0SJp95x5Xsb8KSZrQvb9Y7tLcAyM+sysz7gLuCNtYrLE83emQ0cImla+EvlfOCeOsdU6B7gg+H9B4n6R15RkgT8BFhsZt9usNg6JY0K74cR/Y/3bL1jM7MvmNlkM5tK9N/VH8zsA/WOC0BSu6SO3HuiNv0F9Y7NzF4CVko6LOw6HVhU77gKXMCuZjOof2wvAidIagv/n55ONICiNnHVs3NsML+AtwPPA0uAL9Y5ltuI2ln7iP66+ygwlqhD+W/h55g6xPUmoibF+cC88Hp7g8T2OuCpENsC4Cthf91jy4vxFHYNBqh7XER9IU+H18Lcf/cNEtsMYE749/wvYHQjxBViawNeBkbm7at7bMBXif64WgD8DGipVVw+BY1zzrma8qYz55xzNeWJxjnnXE15onHOOVdTnmicc87VlCca55xzNeWJxjUkSSbp2rztz0m6qkrXvkXSe6pxrQE+57wwk/CsIseuCTPnXrMX150h6e3VibL6JJ2Sm3V6L869XFLbK/V57pXhicY1qiTwLknj6h1IvjBzd6U+CvwfMzu1yLGPA8ea2RV7EcYMoueRKqbIYPj//XKi507cfmQw/IfnhqY00frlnyk8UFgjkbQ9/DxF0qOS7pD0vKSrJb1f0bozz0g6OO8yb5H0v6Hc34fz46GmMVvSfEkfz7vuLEm/BJ4pEs8F4foLJP172PcVogdWry+stUi6B2gH/irpfWGWgjvD586WdFIod7ykP4WJIv+kaG2TZuBrwPsUrW/yPklXSfpc3vUXSJoaXosl/ZBoduopkq7I+365NXjaJf1O0do8CyS9r8h3/LSkReG82/POuylc7ylJ5xQ5r2iZ8Lv+Vvi9zZf0KUmfJpp3a1auFijpTEl/lvSkpF8rmjcvtx7Us5L+CLyr8HNdg6nHk7L+8tdAL2A70fTvy4GRwOeAq8KxW4D35JcNP08BNhNNb94CrAa+Go5dBnw37/z7if7QOoRoNoVW4GLgS6FMC9GT5tPCdbuBaUXinEg0nUcn0eSOfwD+IRx7hGi9j6LfL+/9L4E3hfcHEk3ZQ/j+ifD+LcCd4f2HgP/MO/8q4HN52wuIlo2YSjQD9Alh/5lEyVvhu98LnAy8G/hx3vkji8S7BmgJ70eFn/8X+EBuH9FMGe30n9GgVJlPEM2Bl/t+Y8LP5YRlCIBxwGNAe9j+PPCV8G+1MvzbCbgj93n+asxXAucalJltlfRTogWadlR42mwL05xLWgI8GPY/A+Q3Yd1hZlngb5KWAocT3Yhfl1dbGkl0M0sBT5jZsiKf93rgETPrCp/5C6Kb939VGC9ESeSIaMopAEYomlNsJHCrpEOIpvJp2oNr5qwws7+E92eG11NhezjR9/tf4FuhNnavmf1vkevMB34h6b/Y9d3OJJoANFebaiVKlPlKlXkLcL2ZpQHMbGORzzyBaGHBx8Pvphn4M9G/1TIz+xuAoklHLy5yvmsQnmhco/suUbPPzXn70oRmX0V3oOa8Y8m899m87Sz9/3svnHvJiP46/pSZPZB/QNIpRDWaYootGbGnYsCJZtYvmUr6PjDLzM5VtJ7PIyXO3/n7CFrz3ufHLeAbZvajwgtIOo6o3+cbkh40s68VFHkHUQI9G/iypCPD9d5tZs8VXOuAgs8sVkYMvLSGiNYJuqDg3BkVnOsaiPfRuIYW/tK9g6hjPWc50bK4EK2psTd/6Z8nKRb6baYTrSz4APAJRUsbIOlQRbMUl/NX4M2SxikaKHAB8OgexvIgcGluI9xIIarRrA7vP5RXfhvQkbe9nGhafCQdS9TcV8wDwEfy+jkmSRovaSLQY2Y/J1oM69j8kxQNIphiZrOIFmQbRVQbegD4VEgaSDqmxGcWK/MgcImkRNg/psh3+wtwkqRXhzJtkg4lmghymnb1ufVLRK7xeKJxg8G1RO31OT8murk/QbT8cqnaRjnPESWE3wOXmFkvcCPR9PJPSloA/IgBav2hme4LwCyiWY2fNLM9nVr908DM0Cm+CLgk7P8mUQ3jcaI13XNmETW1zQsd93cCYxStFvoJon6QYrE+SNQf9GdJzxCt29IBvBZ4Ipz/ReBfC06NAz8P5zwFfMeiNXy+TpTk54ff19eLfGypMjcS9W3Nl/Q0cGHYfwPwe0mzQnPkh4DbJM0nSjyHh3+ri4HfhcEAK4p9X9c4fPZm55xzNeU1GuecczXlicY551xNeaJxzjlXU55onHPO1ZQnGuecczXlicY551xNeaJxzjlXU/8PhcoNT09yNjgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.xlabel(\"Number of features selected\")\n",
    "plt.ylabel(\"Cross validation score \\n of number of selected features\")\n",
    "plt.plot(range(1, len(rfecv1.grid_scores_) + 1), rfecv1.grid_scores_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                        fraud_label\n",
       "1                address_lag30_count\n",
       "2            fulladdress_lag30_count\n",
       "3            fulladdress_#days_since\n",
       "4                address_#days_since\n",
       "                   ...              \n",
       "76             address_lag1_lag7_avg\n",
       "77              homephone_lag3_count\n",
       "78    address_name_0_day_cross_count\n",
       "79     address_dob_0_day_cross_count\n",
       "80     address_ssn_0_day_cross_count\n",
       "Name: Variable, Length: 81, dtype: object"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "var_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ranking</th>\n",
       "      <th>variable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>address_dob_14_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>address_dob_1_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>address_dob_30_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>address_dob_3_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>address_dob_7_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>address_homephone_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>address_homephone_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag1_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag1_lag7_avg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag3_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_14_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_1_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_30_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_3_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_7_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>address_ssn_14_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>address_ssn_30_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>address_ssn_3_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1</td>\n",
       "      <td>address_ssn_7_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1</td>\n",
       "      <td>firstname_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>fraud_label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "      <td>fulladdress_lag0_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>fulladdress_lag1_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1</td>\n",
       "      <td>fulladdress_lag1_lag7_avg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>fulladdress_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>homephone_fulladdress_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>homephone_fulladdress_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1</td>\n",
       "      <td>lastname_dob_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1</td>\n",
       "      <td>lastname_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1</td>\n",
       "      <td>name_dob_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_dob_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_firstname_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_firstname_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_lastname_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_lastname_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_name_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_name_dob_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1</td>\n",
       "      <td>zip5_homephone_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>1</td>\n",
       "      <td>zip5_homephone_lag7_count</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ranking                           variable\n",
       "0         1     address_dob_14_day_cross_count\n",
       "1         1      address_dob_1_day_cross_count\n",
       "2         1     address_dob_30_day_cross_count\n",
       "3         1      address_dob_3_day_cross_count\n",
       "4         1      address_dob_7_day_cross_count\n",
       "5         1      address_homephone_#days_since\n",
       "6         1       address_homephone_lag7_count\n",
       "7         1                address_lag14_count\n",
       "8         1                 address_lag1_count\n",
       "9         1              address_lag1_lag7_avg\n",
       "10        1                address_lag30_count\n",
       "11        1                 address_lag3_count\n",
       "12        1                 address_lag7_count\n",
       "13        1    address_name_14_day_cross_count\n",
       "14        1     address_name_1_day_cross_count\n",
       "15        1    address_name_30_day_cross_count\n",
       "16        1     address_name_3_day_cross_count\n",
       "17        1     address_name_7_day_cross_count\n",
       "18        1     address_ssn_14_day_cross_count\n",
       "19        1     address_ssn_30_day_cross_count\n",
       "20        1      address_ssn_3_day_cross_count\n",
       "21        1      address_ssn_7_day_cross_count\n",
       "22        1          firstname_dob_lag30_count\n",
       "23        1                        fraud_label\n",
       "24        1             fulladdress_lag0_count\n",
       "25        1             fulladdress_lag1_count\n",
       "26        1          fulladdress_lag1_lag7_avg\n",
       "27        1             fulladdress_lag7_count\n",
       "28        1  homephone_fulladdress_#days_since\n",
       "29        1  homephone_fulladdress_lag30_count\n",
       "30        1           lastname_dob_lag14_count\n",
       "31        1           lastname_dob_lag30_count\n",
       "32        1               name_dob_#days_since\n",
       "33        1                    ssn_#days_since\n",
       "34        1                ssn_dob_#days_since\n",
       "35        1                ssn_dob_lag30_count\n",
       "36        1          ssn_firstname_#days_since\n",
       "37        1          ssn_firstname_lag30_count\n",
       "38        1                    ssn_lag30_count\n",
       "39        1           ssn_lastname_#days_since\n",
       "40        1           ssn_lastname_lag14_count\n",
       "41        1           ssn_name_dob_lag30_count\n",
       "42        1            ssn_name_dob_lag7_count\n",
       "43        1         zip5_homephone_#days_since\n",
       "44        1          zip5_homephone_lag7_count"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "var_selected_1 = pd.DataFrame(sorted(zip(map(lambda x: round(x), output1.ranking_), var1.columns)),columns = ['ranking', 'variable'])\n",
    "var_selected_1.head(45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45\n"
     ]
    }
   ],
   "source": [
    "# Pick the 45 variables selected from the first round\n",
    "var_list2 = list(var_list).copy()\n",
    "var_list2.remove('fraud_label')\n",
    "var_list2 = np.array(var_list2)[output1.get_support()]\n",
    "print(len(var_list2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "var2 = tt.loc[:,var_list2]\n",
    "var2['fraud_label'] = tt.loc[:,'fraud_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address_lag30_count</th>\n",
       "      <th>fulladdress_lag30_count</th>\n",
       "      <th>address_lag14_count</th>\n",
       "      <th>address_lag7_count</th>\n",
       "      <th>address_ssn_30_day_cross_count</th>\n",
       "      <th>address_name_30_day_cross_count</th>\n",
       "      <th>fulladdress_lag3_count</th>\n",
       "      <th>address_ssn_14_day_cross_count</th>\n",
       "      <th>address_name_14_day_cross_count</th>\n",
       "      <th>address_dob_30_day_cross_count</th>\n",
       "      <th>...</th>\n",
       "      <th>lastname_dob_lag14_count</th>\n",
       "      <th>ssn_name_dob_lag14_count</th>\n",
       "      <th>zip5_homephone_lag7_count</th>\n",
       "      <th>address_homephone_lag7_count</th>\n",
       "      <th>homephone_fulladdress_lag7_count</th>\n",
       "      <th>name_lag14_count</th>\n",
       "      <th>fulladdress_lag1_lag7_avg</th>\n",
       "      <th>address_lag1_lag7_avg</th>\n",
       "      <th>homephone_lag3_count</th>\n",
       "      <th>fraud_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>38511</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38512</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38513</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38514</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38515</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833502</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833503</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833504</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833505</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833506</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>794996 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        address_lag30_count  fulladdress_lag30_count  address_lag14_count  \\\n",
       "38511                     1                        1                    1   \n",
       "38512                     1                        1                    1   \n",
       "38513                     1                        1                    1   \n",
       "38514                     1                        1                    1   \n",
       "38515                     1                        1                    1   \n",
       "...                     ...                      ...                  ...   \n",
       "833502                    1                        1                    1   \n",
       "833503                    1                        1                    1   \n",
       "833504                    1                        1                    1   \n",
       "833505                    1                        1                    1   \n",
       "833506                    1                        1                    1   \n",
       "\n",
       "        address_lag7_count  address_ssn_30_day_cross_count  \\\n",
       "38511                    1                               1   \n",
       "38512                    1                               1   \n",
       "38513                    1                               1   \n",
       "38514                    1                               1   \n",
       "38515                    1                               1   \n",
       "...                    ...                             ...   \n",
       "833502                   1                               1   \n",
       "833503                   1                               1   \n",
       "833504                   1                               1   \n",
       "833505                   1                               1   \n",
       "833506                   1                               1   \n",
       "\n",
       "        address_name_30_day_cross_count  fulladdress_lag3_count  \\\n",
       "38511                                 1                       1   \n",
       "38512                                 1                       1   \n",
       "38513                                 1                       1   \n",
       "38514                                 1                       1   \n",
       "38515                                 1                       1   \n",
       "...                                 ...                     ...   \n",
       "833502                                1                       1   \n",
       "833503                                1                       1   \n",
       "833504                                1                       1   \n",
       "833505                                1                       1   \n",
       "833506                                1                       1   \n",
       "\n",
       "        address_ssn_14_day_cross_count  address_name_14_day_cross_count  \\\n",
       "38511                                1                                1   \n",
       "38512                                1                                1   \n",
       "38513                                1                                1   \n",
       "38514                                1                                1   \n",
       "38515                                1                                1   \n",
       "...                                ...                              ...   \n",
       "833502                               1                                1   \n",
       "833503                               1                                1   \n",
       "833504                               1                                1   \n",
       "833505                               1                                1   \n",
       "833506                               1                                1   \n",
       "\n",
       "        address_dob_30_day_cross_count  ...  lastname_dob_lag14_count  \\\n",
       "38511                                1  ...                         1   \n",
       "38512                                1  ...                         1   \n",
       "38513                                1  ...                         1   \n",
       "38514                                1  ...                         1   \n",
       "38515                                1  ...                         1   \n",
       "...                                ...  ...                       ...   \n",
       "833502                               1  ...                         1   \n",
       "833503                               1  ...                         1   \n",
       "833504                               1  ...                         1   \n",
       "833505                               1  ...                         1   \n",
       "833506                               1  ...                         1   \n",
       "\n",
       "        ssn_name_dob_lag14_count  zip5_homephone_lag7_count  \\\n",
       "38511                          1                          1   \n",
       "38512                          1                          1   \n",
       "38513                          1                          1   \n",
       "38514                          1                          1   \n",
       "38515                          1                          1   \n",
       "...                          ...                        ...   \n",
       "833502                         1                          1   \n",
       "833503                         1                          1   \n",
       "833504                         1                          1   \n",
       "833505                         1                          1   \n",
       "833506                         1                          1   \n",
       "\n",
       "        address_homephone_lag7_count  homephone_fulladdress_lag7_count  \\\n",
       "38511                              1                                 1   \n",
       "38512                              1                                 1   \n",
       "38513                              1                                 1   \n",
       "38514                              1                                 1   \n",
       "38515                              1                                 1   \n",
       "...                              ...                               ...   \n",
       "833502                             1                                 1   \n",
       "833503                             1                                 1   \n",
       "833504                             1                                 1   \n",
       "833505                             1                                 1   \n",
       "833506                             1                                 1   \n",
       "\n",
       "        name_lag14_count  fulladdress_lag1_lag7_avg  address_lag1_lag7_avg  \\\n",
       "38511                  1                        7.0                    7.0   \n",
       "38512                  1                        7.0                    7.0   \n",
       "38513                  1                        7.0                    7.0   \n",
       "38514                  1                        7.0                    7.0   \n",
       "38515                  1                        7.0                    7.0   \n",
       "...                  ...                        ...                    ...   \n",
       "833502                 1                        7.0                    7.0   \n",
       "833503                 1                        7.0                    7.0   \n",
       "833504                 1                        7.0                    7.0   \n",
       "833505                 1                        7.0                    7.0   \n",
       "833506                 1                        7.0                    7.0   \n",
       "\n",
       "        homephone_lag3_count  fraud_label  \n",
       "38511                      1            0  \n",
       "38512                      2            0  \n",
       "38513                      1            0  \n",
       "38514                      1            0  \n",
       "38515                      1            0  \n",
       "...                      ...          ...  \n",
       "833502                     2            0  \n",
       "833503                     2            0  \n",
       "833504                     1            0  \n",
       "833505                     3            0  \n",
       "833506                     2            0  \n",
       "\n",
       "[794996 rows x 46 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "var2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6min 45s, sys: 31.7 s, total: 7min 17s\n",
      "Wall time: 1min 53s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yli/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Second round wrapper (first time)\n",
    "model = LogisticRegression(penalty = 'l2', class_weight = 'balanced')\n",
    "X = var2.iloc[:,:45]\n",
    "y = var2.loc[:,'fraud_label']\n",
    "rfe = RFE(estimator=model, n_features_to_select = 30, step=1)\n",
    "output2 = rfe.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of variables selected: 30\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of variables selected: {output2.get_support().sum()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  1,  1,  1,  1,  1, 10,  1,  1,  8,  2, 12,  1,  5,  4, 11,  6,\n",
       "        1,  1,  3,  1,  7,  1,  1,  1,  1, 14,  1,  1, 16, 13,  1,  1,  1,\n",
       "        1,  1,  1,  1,  1,  9,  1, 15,  1,  1,  1])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output2.ranking_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ranking</th>\n",
       "      <th>variable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>address_dob_1_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>address_homephone_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag1_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag1_lag7_avg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>address_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_14_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_30_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>address_name_7_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>address_ssn_14_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>address_ssn_30_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>fulladdress_lag1_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>fulladdress_lag1_lag7_avg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>fulladdress_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>homephone_fulladdress_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>homephone_fulladdress_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>homephone_lag3_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>lastname_dob_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>lastname_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>name_dob_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1</td>\n",
       "      <td>name_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_dob_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_firstname_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_name_dob_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>ssn_name_dob_lag30_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>zip5_homephone_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>zip5_homephone_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2</td>\n",
       "      <td>address_dob_14_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>3</td>\n",
       "      <td>address_name_1_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>4</td>\n",
       "      <td>address_ssn_3_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>5</td>\n",
       "      <td>address_dob_7_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>6</td>\n",
       "      <td>address_name_3_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>7</td>\n",
       "      <td>address_ssn_1_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>8</td>\n",
       "      <td>address_dob_30_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>9</td>\n",
       "      <td>address_homephone_lag7_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>10</td>\n",
       "      <td>fulladdress_lag3_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>11</td>\n",
       "      <td>address_dob_3_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>12</td>\n",
       "      <td>address_ssn_7_day_cross_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>13</td>\n",
       "      <td>homephone_fulladdress_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>14</td>\n",
       "      <td>address_homephone_#days_since</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>15</td>\n",
       "      <td>name_lag14_count</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>16</td>\n",
       "      <td>ssn_lastname_lag30_count</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ranking                           variable\n",
       "0         1      address_dob_1_day_cross_count\n",
       "1         1      address_homephone_lag14_count\n",
       "2         1                address_lag14_count\n",
       "3         1                 address_lag1_count\n",
       "4         1              address_lag1_lag7_avg\n",
       "5         1                address_lag30_count\n",
       "6         1                 address_lag7_count\n",
       "7         1    address_name_14_day_cross_count\n",
       "8         1    address_name_30_day_cross_count\n",
       "9         1     address_name_7_day_cross_count\n",
       "10        1     address_ssn_14_day_cross_count\n",
       "11        1     address_ssn_30_day_cross_count\n",
       "12        1             fulladdress_lag1_count\n",
       "13        1          fulladdress_lag1_lag7_avg\n",
       "14        1            fulladdress_lag30_count\n",
       "15        1  homephone_fulladdress_lag14_count\n",
       "16        1   homephone_fulladdress_lag7_count\n",
       "17        1               homephone_lag3_count\n",
       "18        1           lastname_dob_lag14_count\n",
       "19        1           lastname_dob_lag30_count\n",
       "20        1               name_dob_lag14_count\n",
       "21        1               name_dob_lag30_count\n",
       "22        1                ssn_dob_lag14_count\n",
       "23        1                ssn_dob_lag30_count\n",
       "24        1          ssn_firstname_lag30_count\n",
       "25        1                    ssn_lag30_count\n",
       "26        1           ssn_name_dob_lag14_count\n",
       "27        1           ssn_name_dob_lag30_count\n",
       "28        1         zip5_homephone_lag14_count\n",
       "29        1          zip5_homephone_lag7_count\n",
       "30        2     address_dob_14_day_cross_count\n",
       "31        3     address_name_1_day_cross_count\n",
       "32        4      address_ssn_3_day_cross_count\n",
       "33        5      address_dob_7_day_cross_count\n",
       "34        6     address_name_3_day_cross_count\n",
       "35        7      address_ssn_1_day_cross_count\n",
       "36        8     address_dob_30_day_cross_count\n",
       "37        9       address_homephone_lag7_count\n",
       "38       10             fulladdress_lag3_count\n",
       "39       11      address_dob_3_day_cross_count\n",
       "40       12      address_ssn_7_day_cross_count\n",
       "41       13  homephone_fulladdress_#days_since\n",
       "42       14      address_homephone_#days_since\n",
       "43       15                   name_lag14_count\n",
       "44       16           ssn_lastname_lag30_count"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Top variables \n",
    "var_selected = pd.DataFrame(sorted(zip(map(lambda x: round(x), output2.ranking_), var2.columns)),columns = ['ranking', 'variable'])\n",
    "var_selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['address_dob_1_day_cross_count',\n",
       " 'address_homephone_lag14_count',\n",
       " 'address_lag14_count',\n",
       " 'address_lag1_count',\n",
       " 'address_lag1_lag7_avg',\n",
       " 'address_lag30_count',\n",
       " 'address_lag7_count',\n",
       " 'address_name_14_day_cross_count',\n",
       " 'address_name_30_day_cross_count',\n",
       " 'address_name_7_day_cross_count',\n",
       " 'address_ssn_14_day_cross_count',\n",
       " 'address_ssn_30_day_cross_count',\n",
       " 'fulladdress_lag1_count',\n",
       " 'fulladdress_lag1_lag7_avg',\n",
       " 'fulladdress_lag30_count',\n",
       " 'homephone_fulladdress_lag14_count',\n",
       " 'homephone_fulladdress_lag7_count',\n",
       " 'homephone_lag3_count',\n",
       " 'lastname_dob_lag14_count',\n",
       " 'lastname_dob_lag30_count',\n",
       " 'name_dob_lag14_count',\n",
       " 'name_dob_lag30_count',\n",
       " 'ssn_dob_lag14_count',\n",
       " 'ssn_dob_lag30_count',\n",
       " 'ssn_firstname_lag30_count',\n",
       " 'ssn_lag30_count',\n",
       " 'ssn_name_dob_lag14_count',\n",
       " 'ssn_name_dob_lag30_count',\n",
       " 'zip5_homephone_lag14_count',\n",
       " 'zip5_homephone_lag7_count']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top30vars = list(var_selected.loc[:29,'variable'].values)\n",
    "top30vars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Output selected features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['address_dob_1_day_cross_count',\n",
       " 'address_homephone_lag14_count',\n",
       " 'address_lag14_count',\n",
       " 'address_lag1_count',\n",
       " 'address_lag1_lag7_avg',\n",
       " 'address_lag30_count',\n",
       " 'address_lag7_count',\n",
       " 'address_name_14_day_cross_count',\n",
       " 'address_name_30_day_cross_count',\n",
       " 'address_name_7_day_cross_count',\n",
       " 'address_ssn_14_day_cross_count',\n",
       " 'address_ssn_30_day_cross_count',\n",
       " 'fulladdress_lag1_count',\n",
       " 'fulladdress_lag1_lag7_avg',\n",
       " 'fulladdress_lag30_count',\n",
       " 'homephone_fulladdress_lag14_count',\n",
       " 'homephone_fulladdress_lag7_count',\n",
       " 'homephone_lag3_count',\n",
       " 'lastname_dob_lag14_count',\n",
       " 'lastname_dob_lag30_count',\n",
       " 'name_dob_lag14_count',\n",
       " 'name_dob_lag30_count',\n",
       " 'ssn_dob_lag14_count',\n",
       " 'ssn_dob_lag30_count',\n",
       " 'ssn_firstname_lag30_count',\n",
       " 'ssn_lag30_count',\n",
       " 'ssn_name_dob_lag14_count',\n",
       " 'ssn_name_dob_lag30_count',\n",
       " 'zip5_homephone_lag14_count',\n",
       " 'zip5_homephone_lag7_count',\n",
       " 'date',\n",
       " 'fraud_label']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top30vars.append('date')\n",
    "top30vars.append('fraud_label')\n",
    "top30vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 825 ms, sys: 94.8 ms, total: 920 ms\n",
      "Wall time: 948 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# oot data\n",
    "oot_data = oot.loc[:,top30vars]\n",
    "oot.loc[:,top30vars].to_csv('oot_after_feature_selection.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.06 s, sys: 203 ms, total: 4.26 s\n",
      "Wall time: 4.28 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# tt data\n",
    "tt_data = tt.loc[:,top30vars]\n",
    "tt.loc[:,top30vars].to_csv('tt_after_feature_selection.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}