Main_Large_Dataset_Work.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest, f_classif, chi2, f_regression\n",
    "import pickle\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression, LinearRegression\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, KFold\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.feature_selection import SelectFromModel, SelectPercentile\n",
    "import csv\n",
    "import pandas as pd \n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn import svm\n",
    "import numpy as np\n",
    "import psycopg2 as pg2\n",
    "from psycopg2.extras import RealDictCursor\n",
    "from sklearn.tree import DecisionTreeRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "con = pg2.connect(host='34.211.227.227',\n",
    "                  dbname='postgres',\n",
    "                  user='postgres')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cur = con.cursor(cursor_factory=RealDictCursor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cur.execute('SELECT * FROM madelon LIMIT 6506;')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results_set2 = cur.fetchall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "con.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'results_set2' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-b35c38acaf11>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresults_set2df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults_set2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mresults_set2df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'resultsset2df.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'results_set2' is not defined"
     ]
    }
   ],
   "source": [
    "results_set2df = pd.DataFrame(results_set2)\n",
    "results_set2df.to_csv('resultsset2df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_set = pd.read_csv('resultsset2df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = results_set['target']\n",
    "X = results_set.drop(['target'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "knearest_pipe = make_pipeline(StandardScaler(), KNeighborsRegressor(n_neighbors=2))\n",
    "knearest_data = knearest_pipe.fit(X,target)\n",
    "knearest_score = knearest_pipe.score(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.50798071926267796"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knearest_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logbig_pipe = make_pipeline(StandardScaler(), LogisticRegression())\n",
    "log_big = logbig_pipe.fit(X, target)\n",
    "logbig_score = log_big.score(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.68936366430986784"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logbig_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Pipeline' object has no attribute 'score_'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-dec019571bbe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mpca_pipe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_pipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mStandardScaler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPCA\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_components\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mpca_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpca_pipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mpca_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpca_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscore_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'Pipeline' object has no attribute 'score_'"
     ]
    }
   ],
   "source": [
    "pca_pipe = make_pipeline(StandardScaler(), PCA(n_components=5))\n",
    "pca_data = pca_pipe.fit(X)\n",
    "pca_scores = pca_data.score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#increasing the max depth give you a better socre, low max, give you a worse score. Closer to one, \n",
    "#means that regression line is close to fitting the data\n",
    "#wanted to see how about 5% of the data, as the leaf nodes would score\n",
    "#having too made leaves and depth can be computation expense so trying to set a max not too high, \n",
    "#that would still give good results\n",
    "tree = DecisionTreeRegressor(max_depth=22, min_samples_leaf=3, max_leaf_nodes=325)\n",
    "#max_depth=22, min_samples_leaf=3, max_leaf_nodes=325"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decisiontree = tree.fit(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = tree.score(X, target)\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "svc = svm.SVC(C=100)\n",
    "svc2 = svm.SVC(C=100)\n",
    "svc_benchmark = svc.fit(X, target)\n",
    "\n",
    "\n",
    "benchmarksvc_score = svc.score(X,target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feat_000</th>\n",
       "      <th>feat_001</th>\n",
       "      <th>feat_002</th>\n",
       "      <th>feat_003</th>\n",
       "      <th>feat_004</th>\n",
       "      <th>feat_005</th>\n",
       "      <th>feat_006</th>\n",
       "      <th>feat_007</th>\n",
       "      <th>feat_008</th>\n",
       "      <th>feat_009</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_991</th>\n",
       "      <th>feat_992</th>\n",
       "      <th>feat_993</th>\n",
       "      <th>feat_994</th>\n",
       "      <th>feat_995</th>\n",
       "      <th>feat_996</th>\n",
       "      <th>feat_997</th>\n",
       "      <th>feat_998</th>\n",
       "      <th>feat_999</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.340098</td>\n",
       "      <td>-1.320514</td>\n",
       "      <td>-1.286333</td>\n",
       "      <td>-1.329391</td>\n",
       "      <td>-0.537599</td>\n",
       "      <td>-0.390216</td>\n",
       "      <td>-0.323870</td>\n",
       "      <td>-0.441399</td>\n",
       "      <td>1.907728</td>\n",
       "      <td>-1.003494</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.938077</td>\n",
       "      <td>1.619856</td>\n",
       "      <td>1.513965</td>\n",
       "      <td>0.136183</td>\n",
       "      <td>0.292799</td>\n",
       "      <td>-0.884726</td>\n",
       "      <td>-0.051330</td>\n",
       "      <td>-0.512662</td>\n",
       "      <td>0.147205</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.585315</td>\n",
       "      <td>-0.275421</td>\n",
       "      <td>1.604859</td>\n",
       "      <td>0.046554</td>\n",
       "      <td>1.894861</td>\n",
       "      <td>-0.581353</td>\n",
       "      <td>1.238764</td>\n",
       "      <td>-0.380432</td>\n",
       "      <td>1.314977</td>\n",
       "      <td>-0.392790</td>\n",
       "      <td>...</td>\n",
       "      <td>0.356865</td>\n",
       "      <td>1.473536</td>\n",
       "      <td>1.033222</td>\n",
       "      <td>0.985235</td>\n",
       "      <td>-1.375029</td>\n",
       "      <td>-0.850786</td>\n",
       "      <td>0.186651</td>\n",
       "      <td>0.025389</td>\n",
       "      <td>0.366575</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.290151</td>\n",
       "      <td>0.076959</td>\n",
       "      <td>0.942821</td>\n",
       "      <td>-0.013861</td>\n",
       "      <td>0.878566</td>\n",
       "      <td>0.275326</td>\n",
       "      <td>-0.772193</td>\n",
       "      <td>0.507709</td>\n",
       "      <td>0.874941</td>\n",
       "      <td>0.653910</td>\n",
       "      <td>...</td>\n",
       "      <td>1.326185</td>\n",
       "      <td>-1.003318</td>\n",
       "      <td>1.453657</td>\n",
       "      <td>0.402349</td>\n",
       "      <td>0.498104</td>\n",
       "      <td>2.010504</td>\n",
       "      <td>0.205830</td>\n",
       "      <td>-0.866195</td>\n",
       "      <td>-0.598564</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.408467</td>\n",
       "      <td>0.011884</td>\n",
       "      <td>1.126055</td>\n",
       "      <td>-0.562285</td>\n",
       "      <td>0.177906</td>\n",
       "      <td>1.183975</td>\n",
       "      <td>1.516265</td>\n",
       "      <td>1.287770</td>\n",
       "      <td>-1.666839</td>\n",
       "      <td>0.810799</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.205146</td>\n",
       "      <td>-0.270598</td>\n",
       "      <td>0.459224</td>\n",
       "      <td>-1.055645</td>\n",
       "      <td>-0.044368</td>\n",
       "      <td>-0.555520</td>\n",
       "      <td>0.510221</td>\n",
       "      <td>0.105843</td>\n",
       "      <td>-0.495197</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.293276</td>\n",
       "      <td>0.006408</td>\n",
       "      <td>0.108143</td>\n",
       "      <td>0.051391</td>\n",
       "      <td>0.476253</td>\n",
       "      <td>0.376401</td>\n",
       "      <td>1.068163</td>\n",
       "      <td>1.370311</td>\n",
       "      <td>0.330304</td>\n",
       "      <td>-1.565246</td>\n",
       "      <td>...</td>\n",
       "      <td>0.288483</td>\n",
       "      <td>-0.204263</td>\n",
       "      <td>-0.584622</td>\n",
       "      <td>0.595530</td>\n",
       "      <td>-1.334408</td>\n",
       "      <td>-0.487893</td>\n",
       "      <td>0.745517</td>\n",
       "      <td>-1.422936</td>\n",
       "      <td>-0.285641</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   feat_000  feat_001  feat_002  feat_003  feat_004  feat_005  feat_006  \\\n",
       "0 -1.340098 -1.320514 -1.286333 -1.329391 -0.537599 -0.390216 -0.323870   \n",
       "1  0.585315 -0.275421  1.604859  0.046554  1.894861 -0.581353  1.238764   \n",
       "2  0.290151  0.076959  0.942821 -0.013861  0.878566  0.275326 -0.772193   \n",
       "3 -0.408467  0.011884  1.126055 -0.562285  0.177906  1.183975  1.516265   \n",
       "4 -0.293276  0.006408  0.108143  0.051391  0.476253  0.376401  1.068163   \n",
       "\n",
       "   feat_007  feat_008  feat_009   ...    feat_991  feat_992  feat_993  \\\n",
       "0 -0.441399  1.907728 -1.003494   ...   -1.938077  1.619856  1.513965   \n",
       "1 -0.380432  1.314977 -0.392790   ...    0.356865  1.473536  1.033222   \n",
       "2  0.507709  0.874941  0.653910   ...    1.326185 -1.003318  1.453657   \n",
       "3  1.287770 -1.666839  0.810799   ...   -0.205146 -0.270598  0.459224   \n",
       "4  1.370311  0.330304 -1.565246   ...    0.288483 -0.204263 -0.584622   \n",
       "\n",
       "   feat_994  feat_995  feat_996  feat_997  feat_998  feat_999  target  \n",
       "0  0.136183  0.292799 -0.884726 -0.051330 -0.512662  0.147205       1  \n",
       "1  0.985235 -1.375029 -0.850786  0.186651  0.025389  0.366575       0  \n",
       "2  0.402349  0.498104  2.010504  0.205830 -0.866195 -0.598564       0  \n",
       "3 -1.055645 -0.044368 -0.555520  0.510221  0.105843 -0.495197       0  \n",
       "4  0.595530 -1.334408 -0.487893  0.745517 -1.422936 -0.285641       0  \n",
       "\n",
       "[5 rows x 1001 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_set.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "results_set.drop(['_id'], axis=1, inplace=True)\n",
    "#results_set.drop(['target'], axis=1, inplace=True)\n",
    "results_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#goes through and test the data\n",
    "def calculate_r2_for_feature(df, model): \n",
    "    scores_list = []\n",
    "    for col in df.columns:\n",
    "        \n",
    "        X = df.drop(col, axis = 1)\n",
    "        y = df[col]\n",
    "    \n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)\n",
    "    \n",
    "    \n",
    "        regressor = model()\n",
    "        regressor.fit(X, y)\n",
    "    \n",
    "        score = regressor.score(X, y)\n",
    "        scores_list.append(score)\n",
    "    return scores_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feat_000</th>\n",
       "      <th>feat_001</th>\n",
       "      <th>feat_002</th>\n",
       "      <th>feat_003</th>\n",
       "      <th>feat_004</th>\n",
       "      <th>feat_005</th>\n",
       "      <th>feat_006</th>\n",
       "      <th>feat_007</th>\n",
       "      <th>feat_008</th>\n",
       "      <th>feat_009</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_990</th>\n",
       "      <th>feat_991</th>\n",
       "      <th>feat_992</th>\n",
       "      <th>feat_993</th>\n",
       "      <th>feat_994</th>\n",
       "      <th>feat_995</th>\n",
       "      <th>feat_996</th>\n",
       "      <th>feat_997</th>\n",
       "      <th>feat_998</th>\n",
       "      <th>feat_999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.340098</td>\n",
       "      <td>-1.320514</td>\n",
       "      <td>-1.286333</td>\n",
       "      <td>-1.329391</td>\n",
       "      <td>-0.537599</td>\n",
       "      <td>-0.390216</td>\n",
       "      <td>-0.323870</td>\n",
       "      <td>-0.441399</td>\n",
       "      <td>1.907728</td>\n",
       "      <td>-1.003494</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.511640</td>\n",
       "      <td>-1.938077</td>\n",
       "      <td>1.619856</td>\n",
       "      <td>1.513965</td>\n",
       "      <td>0.136183</td>\n",
       "      <td>0.292799</td>\n",
       "      <td>-0.884726</td>\n",
       "      <td>-0.051330</td>\n",
       "      <td>-0.512662</td>\n",
       "      <td>0.147205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.585315</td>\n",
       "      <td>-0.275421</td>\n",
       "      <td>1.604859</td>\n",
       "      <td>0.046554</td>\n",
       "      <td>1.894861</td>\n",
       "      <td>-0.581353</td>\n",
       "      <td>1.238764</td>\n",
       "      <td>-0.380432</td>\n",
       "      <td>1.314977</td>\n",
       "      <td>-0.392790</td>\n",
       "      <td>...</td>\n",
       "      <td>0.380560</td>\n",
       "      <td>0.356865</td>\n",
       "      <td>1.473536</td>\n",
       "      <td>1.033222</td>\n",
       "      <td>0.985235</td>\n",
       "      <td>-1.375029</td>\n",
       "      <td>-0.850786</td>\n",
       "      <td>0.186651</td>\n",
       "      <td>0.025389</td>\n",
       "      <td>0.366575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.290151</td>\n",
       "      <td>0.076959</td>\n",
       "      <td>0.942821</td>\n",
       "      <td>-0.013861</td>\n",
       "      <td>0.878566</td>\n",
       "      <td>0.275326</td>\n",
       "      <td>-0.772193</td>\n",
       "      <td>0.507709</td>\n",
       "      <td>0.874941</td>\n",
       "      <td>0.653910</td>\n",
       "      <td>...</td>\n",
       "      <td>1.633663</td>\n",
       "      <td>1.326185</td>\n",
       "      <td>-1.003318</td>\n",
       "      <td>1.453657</td>\n",
       "      <td>0.402349</td>\n",
       "      <td>0.498104</td>\n",
       "      <td>2.010504</td>\n",
       "      <td>0.205830</td>\n",
       "      <td>-0.866195</td>\n",
       "      <td>-0.598564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.408467</td>\n",
       "      <td>0.011884</td>\n",
       "      <td>1.126055</td>\n",
       "      <td>-0.562285</td>\n",
       "      <td>0.177906</td>\n",
       "      <td>1.183975</td>\n",
       "      <td>1.516265</td>\n",
       "      <td>1.287770</td>\n",
       "      <td>-1.666839</td>\n",
       "      <td>0.810799</td>\n",
       "      <td>...</td>\n",
       "      <td>0.758229</td>\n",
       "      <td>-0.205146</td>\n",
       "      <td>-0.270598</td>\n",
       "      <td>0.459224</td>\n",
       "      <td>-1.055645</td>\n",
       "      <td>-0.044368</td>\n",
       "      <td>-0.555520</td>\n",
       "      <td>0.510221</td>\n",
       "      <td>0.105843</td>\n",
       "      <td>-0.495197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.293276</td>\n",
       "      <td>0.006408</td>\n",
       "      <td>0.108143</td>\n",
       "      <td>0.051391</td>\n",
       "      <td>0.476253</td>\n",
       "      <td>0.376401</td>\n",
       "      <td>1.068163</td>\n",
       "      <td>1.370311</td>\n",
       "      <td>0.330304</td>\n",
       "      <td>-1.565246</td>\n",
       "      <td>...</td>\n",
       "      <td>0.137683</td>\n",
       "      <td>0.288483</td>\n",
       "      <td>-0.204263</td>\n",
       "      <td>-0.584622</td>\n",
       "      <td>0.595530</td>\n",
       "      <td>-1.334408</td>\n",
       "      <td>-0.487893</td>\n",
       "      <td>0.745517</td>\n",
       "      <td>-1.422936</td>\n",
       "      <td>-0.285641</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   feat_000  feat_001  feat_002  feat_003  feat_004  feat_005  feat_006  \\\n",
       "0 -1.340098 -1.320514 -1.286333 -1.329391 -0.537599 -0.390216 -0.323870   \n",
       "1  0.585315 -0.275421  1.604859  0.046554  1.894861 -0.581353  1.238764   \n",
       "2  0.290151  0.076959  0.942821 -0.013861  0.878566  0.275326 -0.772193   \n",
       "3 -0.408467  0.011884  1.126055 -0.562285  0.177906  1.183975  1.516265   \n",
       "4 -0.293276  0.006408  0.108143  0.051391  0.476253  0.376401  1.068163   \n",
       "\n",
       "   feat_007  feat_008  feat_009    ...     feat_990  feat_991  feat_992  \\\n",
       "0 -0.441399  1.907728 -1.003494    ...    -0.511640 -1.938077  1.619856   \n",
       "1 -0.380432  1.314977 -0.392790    ...     0.380560  0.356865  1.473536   \n",
       "2  0.507709  0.874941  0.653910    ...     1.633663  1.326185 -1.003318   \n",
       "3  1.287770 -1.666839  0.810799    ...     0.758229 -0.205146 -0.270598   \n",
       "4  1.370311  0.330304 -1.565246    ...     0.137683  0.288483 -0.204263   \n",
       "\n",
       "   feat_993  feat_994  feat_995  feat_996  feat_997  feat_998  feat_999  \n",
       "0  1.513965  0.136183  0.292799 -0.884726 -0.051330 -0.512662  0.147205  \n",
       "1  1.033222  0.985235 -1.375029 -0.850786  0.186651  0.025389  0.366575  \n",
       "2  1.453657  0.402349  0.498104  2.010504  0.205830 -0.866195 -0.598564  \n",
       "3  0.459224 -1.055645 -0.044368 -0.555520  0.510221  0.105843 -0.495197  \n",
       "4 -0.584622  0.595530 -1.334408 -0.487893  0.745517 -1.422936 -0.285641  \n",
       "\n",
       "[5 rows x 1000 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset = results_set[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#results_set[results_set.columns[:200]]\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.DataFrame(calculate_r2_for_feature(results_set, KNeighborsRegressor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df.sort_values(0, ascending=False, inplace=True)\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results_df = pd.DataFrame(calculate_r2_for_feature(results_set[:,:50], DecisionTreeRegressor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'results_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-a87488251595>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#results_df.sort_values(0, ascending=True, inplace=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mresults_df\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'results_df' is not defined"
     ]
    }
   ],
   "source": [
    "#results_df.sort_values(0, ascending=True, inplace=True)\n",
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SelectKBest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#farther away from 1 the better the correlation/importance\n",
    "select_k = SelectKBest(score_func = f_classif, k=100)\n",
    "k_select = select_k.fit_transform(X, target)\n",
    "df = pd.DataFrame(select_k.scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>275.359632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683</th>\n",
       "      <td>189.231256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>703</th>\n",
       "      <td>175.197028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>922</th>\n",
       "      <td>151.214517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              0\n",
       "271  275.359632\n",
       "683  189.231256\n",
       "703  175.197028\n",
       "922  151.214517"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values(0, ascending=False, inplace=True)\n",
    "select_k = df.iloc[0:4]\n",
    "select_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>275.359632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683</th>\n",
       "      <td>189.231256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>703</th>\n",
       "      <td>175.197028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>922</th>\n",
       "      <td>151.214517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              0\n",
       "271  275.359632\n",
       "683  189.231256\n",
       "703  175.197028\n",
       "922  151.214517"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#bigger difference with numbers seem to narrow it down more to stronger features\n",
    "#big difference in the first but also after 922, it drops significantlly \n",
    "#got both results as top 5 with second biggest difference with performing this test\n",
    "select_k2 = SelectPercentile(score_func = f_classif, percentile=20)\n",
    "k_select2 = select_k2.fit(X, target)\n",
    "select_per2 = pd.DataFrame(select_k2.scores_).sort_values(0, ascending=False, inplace=False)\n",
    "select_per = select_per2.iloc[0:4]\n",
    "select_per"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = select_k.index \n",
    "final_df = results_set.iloc[mask]     \n",
    "\n",
    "X2 = final_df.drop(['target'], axis=1)\n",
    "target2 = final_df['target']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#less depth, lower score\n",
    "#same for amount of leaves\n",
    "treeregressor = DecisionTreeRegressor(max_depth=20, min_samples_leaf=3, max_leaf_nodes=325)\n",
    "tree_results2 = treeregressor.fit(X, target)\n",
    "score_tree = tree_results2.score(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83468460855036475"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}