Project 3 Cleaning Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest, f_classif, chi2, f_regression\n",
    "import pickle\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression, LinearRegression\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, KFold\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.feature_selection import SelectFromModel, SelectPercentile\n",
    "import csv\n",
    "import pandas as pd \n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm\n",
    "import numpy as np\n",
    "from sklearn.tree import DecisionTreeRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1-Loading and Checking Data of Smaller Dataset ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_traindf = pd.read_csv('madelon_train.data', sep = ' ', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2000, 501)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "madelon_traindf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_traindf.drop(labels=[500], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>490</th>\n",
       "      <th>491</th>\n",
       "      <th>492</th>\n",
       "      <th>493</th>\n",
       "      <th>494</th>\n",
       "      <th>495</th>\n",
       "      <th>496</th>\n",
       "      <th>497</th>\n",
       "      <th>498</th>\n",
       "      <th>499</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>485</td>\n",
       "      <td>477</td>\n",
       "      <td>537</td>\n",
       "      <td>479</td>\n",
       "      <td>452</td>\n",
       "      <td>471</td>\n",
       "      <td>491</td>\n",
       "      <td>476</td>\n",
       "      <td>475</td>\n",
       "      <td>473</td>\n",
       "      <td>...</td>\n",
       "      <td>477</td>\n",
       "      <td>481</td>\n",
       "      <td>477</td>\n",
       "      <td>485</td>\n",
       "      <td>511</td>\n",
       "      <td>485</td>\n",
       "      <td>481</td>\n",
       "      <td>479</td>\n",
       "      <td>475</td>\n",
       "      <td>496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>483</td>\n",
       "      <td>458</td>\n",
       "      <td>460</td>\n",
       "      <td>487</td>\n",
       "      <td>587</td>\n",
       "      <td>475</td>\n",
       "      <td>526</td>\n",
       "      <td>479</td>\n",
       "      <td>485</td>\n",
       "      <td>469</td>\n",
       "      <td>...</td>\n",
       "      <td>463</td>\n",
       "      <td>478</td>\n",
       "      <td>487</td>\n",
       "      <td>338</td>\n",
       "      <td>513</td>\n",
       "      <td>486</td>\n",
       "      <td>483</td>\n",
       "      <td>492</td>\n",
       "      <td>510</td>\n",
       "      <td>517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>487</td>\n",
       "      <td>542</td>\n",
       "      <td>499</td>\n",
       "      <td>468</td>\n",
       "      <td>448</td>\n",
       "      <td>471</td>\n",
       "      <td>442</td>\n",
       "      <td>478</td>\n",
       "      <td>480</td>\n",
       "      <td>477</td>\n",
       "      <td>...</td>\n",
       "      <td>487</td>\n",
       "      <td>481</td>\n",
       "      <td>492</td>\n",
       "      <td>650</td>\n",
       "      <td>506</td>\n",
       "      <td>501</td>\n",
       "      <td>480</td>\n",
       "      <td>489</td>\n",
       "      <td>499</td>\n",
       "      <td>498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>480</td>\n",
       "      <td>491</td>\n",
       "      <td>510</td>\n",
       "      <td>485</td>\n",
       "      <td>495</td>\n",
       "      <td>472</td>\n",
       "      <td>417</td>\n",
       "      <td>474</td>\n",
       "      <td>502</td>\n",
       "      <td>476</td>\n",
       "      <td>...</td>\n",
       "      <td>491</td>\n",
       "      <td>480</td>\n",
       "      <td>474</td>\n",
       "      <td>572</td>\n",
       "      <td>454</td>\n",
       "      <td>469</td>\n",
       "      <td>475</td>\n",
       "      <td>482</td>\n",
       "      <td>494</td>\n",
       "      <td>461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>484</td>\n",
       "      <td>502</td>\n",
       "      <td>528</td>\n",
       "      <td>489</td>\n",
       "      <td>466</td>\n",
       "      <td>481</td>\n",
       "      <td>402</td>\n",
       "      <td>478</td>\n",
       "      <td>487</td>\n",
       "      <td>468</td>\n",
       "      <td>...</td>\n",
       "      <td>488</td>\n",
       "      <td>479</td>\n",
       "      <td>452</td>\n",
       "      <td>435</td>\n",
       "      <td>486</td>\n",
       "      <td>508</td>\n",
       "      <td>481</td>\n",
       "      <td>504</td>\n",
       "      <td>495</td>\n",
       "      <td>511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 500 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0    1    2    3    4    5    6    7    8    9   ...   490  491  492  493  \\\n",
       "0  485  477  537  479  452  471  491  476  475  473 ...   477  481  477  485   \n",
       "1  483  458  460  487  587  475  526  479  485  469 ...   463  478  487  338   \n",
       "2  487  542  499  468  448  471  442  478  480  477 ...   487  481  492  650   \n",
       "3  480  491  510  485  495  472  417  474  502  476 ...   491  480  474  572   \n",
       "4  484  502  528  489  466  481  402  478  487  468 ...   488  479  452  435   \n",
       "\n",
       "   494  495  496  497  498  499  \n",
       "0  511  485  481  479  475  496  \n",
       "1  513  486  483  492  510  517  \n",
       "2  506  501  480  489  499  498  \n",
       "3  454  469  475  482  494  461  \n",
       "4  486  508  481  504  495  511  \n",
       "\n",
       "[5 rows x 500 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "madelon_traindf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_valid = pd.read_csv('madelon_valid.data', sep= ' ', header=None)\n",
    "madelon_valid.drop([500], inplace=True, axis=1)\n",
    "#madelon_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_testdf = pd.read_csv('madelon_test.data', sep= ' ', header=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_testdf.drop([500], inplace=True, axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#need to change the label column name, otherwise it gives me two columns named 0 so try to index it would not end well\n",
    "madelon_trainlablsdf = pd.read_csv('madelon_train.labels', sep=' ', header=None)\n",
    "madelon_trainlablsdf.rename(index=None, columns={0:'label'}, inplace=True)\n",
    "#madelon_trainlablsdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_validlabelsdf = pd.read_csv('madelon_valid.labels', sep=' ', header=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "madelon_valid1 = pd.concat([madelon_validlabelsdf, madelon_valid], axis=1)\n",
    "#madelon_valid1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#madelon_valid1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_traindf1 = pd.concat([madelon_trainlablsdf, madelon_traindf], axis=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Taking 10% percent sample to test the models on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#madelon_traindf1.info()\n",
    "train_set = madelon_traindf1.sample(200) #replace=True)\n",
    "train_set2 = madelon_traindf1.sample(200)\n",
    "train_set3 = madelon_traindf1.sample(200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_set3.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_set = madelon_testdf.sample(200)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "valid_set = madelon_valid1.sample(200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#target_valid = valid_set['label']\n",
    "#target_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "values = madelon_traindf.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target = train_set['label']\n",
    "X = train_set.drop(['label'], axis=1)\n",
    "Z = test_set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark Models ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Starting with benchmark models to get a basic idea of how each model does with the data.\n",
    "\n",
    "I am using pipelines to make sure to simplify the process. \n",
    "\n",
    "After fitting the data to the model, I then scored the model to see how well it preformed. \n",
    "\n",
    "The Madelon Set that was import, included a train and test set already, so I need not need to do a train test split.\n",
    "\n",
    "The label data that was imported is target that is used in the all the models to see the correlation to the features in the dataset to that of the target/label. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Train dataset\n",
    "logistic_pipeline = make_pipeline(StandardScaler(), LogisticRegression())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logistic = logistic_pipeline.fit(X, target)\n",
    "logistictrain_score = logistic.score(X,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistictrain_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Test dataset\n",
    "logistic_pipeline2 = make_pipeline(StandardScaler(), LogisticRegression())\n",
    "logistic2 = logistic_pipeline2.fit(Z, target)\n",
    "logisticttest_score = logistic2.score(Z,target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNeighbors Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('standardscaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('kneighborsregressor', KNeighborsRegressor(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "          metric_params=None, n_jobs=1, n_neighbors=2, p=2,\n",
       "          weights='uniform'))])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#going with more neighbors have me a lower score\n",
    "knearest_pipeline = make_pipeline(StandardScaler(), KNeighborsRegressor(n_neighbors=2))\n",
    "knearest_pipeline2 = make_pipeline(StandardScaler(), KNeighborsRegressor(n_neighbors=2))\n",
    "knearest_pipeline.fit(X,target)\n",
    "knearest_pipeline2.fit(Z,target)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47089947089947093"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knearesttrain_score = knearest_pipeline.score(X,target)\n",
    "knearesttest_score2 = knearest_pipeline2.score(Z,target)\n",
    "knearesttrain_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.51758481170245885"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knearesttest_score2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('standardscaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('pca', PCA(copy=True, iterated_power='auto', n_components=195, random_state=None,\n",
       "  svd_solver='auto', tol=0.0, whiten=False))])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca_pipeline = make_pipeline(StandardScaler(), PCA(n_components=195))\n",
    "pca_pipeline2 = make_pipeline(StandardScaler(), PCA(n_components=195))\n",
    "#train dataset fit\n",
    "pca_pipeline.fit(X)\n",
    "#test dataset fit\n",
    "pca_pipeline2.fit(Z)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-444.94302385223165"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca_score_train = pca_pipeline.score(X,target)\n",
    "pca_score_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-440.06923881013995"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca_score_test = pca_pipeline2.score(Z,target)\n",
    "pca_score_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "svc = svm.SVC(C=100)\n",
    "svc2 = svm.SVC(C=100)\n",
    "svc_train = svc.fit(X, target)\n",
    "\n",
    "\n",
    "trainsvc_score = svc_train.score(X,target)\n",
    "testsvc_score = svc.score(Z,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainsvc_score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DecisionTree Regressor ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#max_depth=22, min_samples_leaf=3, max_leaf_nodes=100\n",
    "regressor = DecisionTreeRegressor(max_depth=22, min_samples_leaf=3, max_leaf_nodes=100)\n",
    "tree_results = regressor.fit(X, target)\n",
    "score = tree_results.score(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.84784037071618767"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "regressor2 = DecisionTreeRegressor(max_depth=22, min_samples_leaf=3, max_leaf_nodes=100)\n",
    "tree_results = regressor2.fit(Z, target)\n",
    "test_score = tree_results2.score(Z, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.75287892934951706"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The scores list creates a dictionary of the scores to be able to view in a dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = []\n",
    "\n",
    "score.append({'logistic train score': logisticttest_score}) \n",
    "score.append({'logistic test score': logistictrain_score}) \n",
    "score.append({'PCA train score': pca_score_test})\n",
    "score.append({'PCA test score':pca_score_train})\n",
    "score.append({'Knearest score train': knearesttrain_score})\n",
    "score.append({'Kneareat score test': knearesttest_score2})\n",
    "score.append({'SVC score train': trainsvc_score})\n",
    "score.append({'SVC score test': testsvc_score})\n",
    "score.append({'DecisionTree Regressor score test': score})\n",
    "score.append({'DecisionTree Regressor score test': test_score})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'logistic train score': 1.0},\n",
       " {'logistic test score': 1.0},\n",
       " {'PCA train score': -440.06923881013995},\n",
       " {'PCA test score': -444.94302385223165},\n",
       " {'Knearest score train': 0.47089947089947093},\n",
       " {'Kneareat score test': 0.51758481170245885},\n",
       " {'SVC score train': 1.0},\n",
       " {'SVC score test': 0.59499999999999997},\n",
       " {'DecisionTree Regressor score test': [...]},\n",
       " {'DecisionTree Regressor score test': -0.75287892934951706}]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The function below takes the r2 score for each of the models.\n",
    "The closer the score get should try to be closer to 1 since have a score of 1 means that model fit perfectly."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Selection using the whole data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calculate_r2_for_feature(df, model): \n",
    "    scores_list = []\n",
    "    for col in df.columns:\n",
    "        \n",
    "        X = df.drop(col, axis = 1)\n",
    "        y = df[col]\n",
    "    \n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)\n",
    "    \n",
    "    \n",
    "        regressor = model()\n",
    "        regressor.fit(X_train, y_train)\n",
    "    \n",
    "        score = regressor.score(X_test, y_test)\n",
    "        scores_list.append(score)\n",
    "    return scores_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Putting the Scores from the function into a dataframe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scores = pd.DataFrame(calculate_r2_for_feature(madelon_traindf, DecisionTreeRegressor))\n",
    "scores = scores.sort_values(0, ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "madelon_traindf.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The scores below go negative after the first 20 features. This means that once the scores go negative, there is little correlation between the feature and the target."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>0.966261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.965643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>0.964777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>0.964617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>0.962284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>0.960907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>472</th>\n",
       "      <td>0.958995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>0.957781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>0.957728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>0.957469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>0.957169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>0.955187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>0.954583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>0.952191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.951794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>0.951445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>0.947507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>0.943041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>0.824425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>0.816780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>-0.648844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>-0.669871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>-0.690929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>-0.704052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>-0.704685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>-0.710873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>-0.713479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>471</th>\n",
       "      <td>-0.720932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>-0.721967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>-0.722636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>-1.308511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>-1.309763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>-1.314838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>-1.319223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>-1.321320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>-1.323243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>-1.323919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>-1.330815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>-1.341248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>-1.346015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>-1.355590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>-1.372366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>-1.378475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>-1.379196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>-1.382158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>-1.394303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>-1.398198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-1.409341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>-1.422622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>-1.426440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>-1.426961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>-1.504477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>-1.520341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>-1.526545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>-1.560658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>410</th>\n",
       "      <td>-1.572498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>-1.598875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>-1.612413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>-1.654180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>-1.741536</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0\n",
       "318  0.966261\n",
       "28   0.965643\n",
       "64   0.964777\n",
       "153  0.964617\n",
       "433  0.962284\n",
       "451  0.960907\n",
       "472  0.958995\n",
       "241  0.957781\n",
       "442  0.957728\n",
       "475  0.957469\n",
       "336  0.957169\n",
       "281  0.955187\n",
       "453  0.954583\n",
       "493  0.952191\n",
       "48   0.951794\n",
       "378  0.951445\n",
       "105  0.947507\n",
       "128  0.943041\n",
       "455  0.824425\n",
       "338  0.816780\n",
       "91  -0.648844\n",
       "250 -0.669871\n",
       "335 -0.690929\n",
       "131 -0.704052\n",
       "222 -0.704685\n",
       "80  -0.710873\n",
       "248 -0.713479\n",
       "471 -0.720932\n",
       "83  -0.721967\n",
       "93  -0.722636\n",
       "..        ...\n",
       "481 -1.308511\n",
       "435 -1.309763\n",
       "397 -1.314838\n",
       "291 -1.319223\n",
       "242 -1.321320\n",
       "117 -1.323243\n",
       "223 -1.323919\n",
       "206 -1.330815\n",
       "372 -1.341248\n",
       "349 -1.346015\n",
       "394 -1.355590\n",
       "303 -1.372366\n",
       "450 -1.378475\n",
       "192 -1.379196\n",
       "163 -1.382158\n",
       "249 -1.394303\n",
       "330 -1.398198\n",
       "14  -1.409341\n",
       "352 -1.422622\n",
       "486 -1.426440\n",
       "320 -1.426961\n",
       "69  -1.504477\n",
       "288 -1.520341\n",
       "417 -1.526545\n",
       "174 -1.560658\n",
       "410 -1.572498\n",
       "260 -1.598875\n",
       "144 -1.612413\n",
       "127 -1.654180\n",
       "264 -1.741536\n",
       "\n",
       "[500 rows x 1 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_scores = scores[scores[0] > 0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##Top 20 features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>0.966261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.965643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>0.964777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>0.964617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>0.962284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>0.960907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>472</th>\n",
       "      <td>0.958995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>0.957781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>0.957728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>0.957469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>0.957169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>0.955187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>0.954583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>0.952191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.951794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>0.951445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>0.947507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>0.943041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>0.824425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>0.816780</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0\n",
       "318  0.966261\n",
       "28   0.965643\n",
       "64   0.964777\n",
       "153  0.964617\n",
       "433  0.962284\n",
       "451  0.960907\n",
       "472  0.958995\n",
       "241  0.957781\n",
       "442  0.957728\n",
       "475  0.957469\n",
       "336  0.957169\n",
       "281  0.955187\n",
       "453  0.954583\n",
       "493  0.952191\n",
       "48   0.951794\n",
       "378  0.951445\n",
       "105  0.947507\n",
       "128  0.943041\n",
       "455  0.824425\n",
       "338  0.816780"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Selection for the random sample of the whole set, to see if it returns the same top 20 features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "score_sample1 = pd.DataFrame(calculate_r2_for_feature(train_set, DecisionTreeRegressor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "score_sample1.sort_values(0, ascending=False, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "score_sample1 = score_sample1.loc[score_sample1[0] > 0]\n",
    "#score_sample1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_set2 = pd.DataFrame(calculate_r2_for_feature(train_set2, DecisionTreeRegressor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>0.965662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>0.950892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.950412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>0.947841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.944132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>0.937618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>0.935185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>0.934149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>0.933272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>0.933042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>0.929892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>0.924412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>473</th>\n",
       "      <td>0.919466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>0.919344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>0.915512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>0.906874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>0.882609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>0.845935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>0.674235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>0.394726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>-0.120886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>-0.181330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>373</th>\n",
       "      <td>-0.187491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>-0.203913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>-0.205133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>-0.210309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416</th>\n",
       "      <td>-0.230889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>-0.246861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>-0.257764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>-0.267269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>-2.097227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>-2.107365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>-2.115622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316</th>\n",
       "      <td>-2.116761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>-2.121367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>-2.136454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>-2.155101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>-2.171495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>-2.180446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>-2.196023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>-2.223983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>-2.231625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-2.233280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>-2.235223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>-2.323303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>-2.350004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>-2.416127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>-2.449190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>-2.481160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>-2.497629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>-2.530915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>-2.875826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>-2.944213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>-2.972954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>-3.192595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>-3.447533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>-3.512485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>-3.586737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>-4.060228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>-4.189873</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>501 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0\n",
       "319  0.965662\n",
       "154  0.950892\n",
       "49   0.950412\n",
       "106  0.947841\n",
       "29   0.944132\n",
       "242  0.937618\n",
       "434  0.935185\n",
       "494  0.934149\n",
       "337  0.933272\n",
       "282  0.933042\n",
       "476  0.929892\n",
       "443  0.924412\n",
       "473  0.919466\n",
       "379  0.919344\n",
       "452  0.915512\n",
       "129  0.906874\n",
       "65   0.882609\n",
       "454  0.845935\n",
       "456  0.674235\n",
       "339  0.394726\n",
       "239 -0.120886\n",
       "409 -0.181330\n",
       "373 -0.187491\n",
       "412 -0.203913\n",
       "230 -0.205133\n",
       "363 -0.210309\n",
       "416 -0.230889\n",
       "54  -0.246861\n",
       "403 -0.257764\n",
       "85  -0.267269\n",
       "..        ...\n",
       "219 -2.097227\n",
       "290 -2.107365\n",
       "152 -2.115622\n",
       "316 -2.116761\n",
       "235 -2.121367\n",
       "202 -2.136454\n",
       "365 -2.155101\n",
       "291 -2.171495\n",
       "283 -2.180446\n",
       "259 -2.196023\n",
       "499 -2.223983\n",
       "205 -2.231625\n",
       "22  -2.233280\n",
       "35  -2.235223\n",
       "396 -2.323303\n",
       "100 -2.350004\n",
       "356 -2.416127\n",
       "149 -2.449190\n",
       "40  -2.481160\n",
       "342 -2.497629\n",
       "119 -2.530915\n",
       "407 -2.875826\n",
       "158 -2.944213\n",
       "43  -2.972954\n",
       "301 -3.192595\n",
       "155 -3.447533\n",
       "258 -3.512485\n",
       "58  -3.586737\n",
       "447 -4.060228\n",
       "424 -4.189873\n",
       "\n",
       "[501 rows x 1 columns]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_set2.sort_values(0, ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sample_set2' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-106-aa4408d2349e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msample2_set\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msample_set2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msample_set2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m#sample2_set\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sample_set2' is not defined"
     ]
    }
   ],
   "source": [
    "sample2_set = sample_set2.loc[sample_set2[0] > 0]\n",
    "sample2_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_sample3 = pd.DataFrame(calculate_r2_for_feature(train_set3, DecisionTreeRegressor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>0.962956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>0.956670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>473</th>\n",
       "      <td>0.953539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>0.951021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>0.946070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.945302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>0.943138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>0.939604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>0.935168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>0.934610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.928394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>0.922672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>0.920123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>0.910216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>0.887766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>0.883231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>0.881253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>0.860483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>0.289215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>0.263362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>-0.030238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>-0.122529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>-0.197300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>-0.205397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>-0.228279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>-0.245716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>-0.246489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>-0.261773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>-0.268541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>-0.275501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>-2.048635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>-2.055382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>-2.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>-2.128902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>-2.133258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>-2.168057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>-2.188988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>-2.193747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>-2.218053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>-2.257960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>-2.258805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>-2.280513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>-2.302653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>-2.305112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357</th>\n",
       "      <td>-2.353431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>-2.394012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>-2.450778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>426</th>\n",
       "      <td>-2.461614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>-2.494633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>-2.513534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>-2.613630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>-2.655185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>-2.715843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>-2.723262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>-2.745614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>-2.982543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-3.008011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>-3.021163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>-3.103251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>-4.609035</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>501 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0\n",
       "65   0.962956\n",
       "476  0.956670\n",
       "473  0.953539\n",
       "494  0.951021\n",
       "454  0.946070\n",
       "29   0.945302\n",
       "319  0.943138\n",
       "434  0.939604\n",
       "337  0.935168\n",
       "379  0.934610\n",
       "49   0.928394\n",
       "443  0.922672\n",
       "452  0.920123\n",
       "106  0.910216\n",
       "154  0.887766\n",
       "242  0.883231\n",
       "282  0.881253\n",
       "129  0.860483\n",
       "339  0.289215\n",
       "456  0.263362\n",
       "382 -0.030238\n",
       "237 -0.122529\n",
       "231 -0.197300\n",
       "72  -0.205397\n",
       "276 -0.228279\n",
       "107 -0.245716\n",
       "141 -0.246489\n",
       "208 -0.261773\n",
       "429 -0.268541\n",
       "447 -0.275501\n",
       "..        ...\n",
       "156 -2.048635\n",
       "346 -2.055382\n",
       "39  -2.090000\n",
       "108 -2.128902\n",
       "43  -2.133258\n",
       "297 -2.168057\n",
       "102 -2.188988\n",
       "174 -2.193747\n",
       "334 -2.218053\n",
       "144 -2.257960\n",
       "491 -2.258805\n",
       "323 -2.280513\n",
       "126 -2.302653\n",
       "111 -2.305112\n",
       "357 -2.353431\n",
       "75  -2.394012\n",
       "131 -2.450778\n",
       "426 -2.461614\n",
       "195 -2.494633\n",
       "261 -2.513534\n",
       "254 -2.613630\n",
       "326 -2.655185\n",
       "175 -2.715843\n",
       "361 -2.723262\n",
       "121 -2.745614\n",
       "149 -2.982543\n",
       "7   -3.008011\n",
       "165 -3.021163\n",
       "485 -3.103251\n",
       "363 -4.609035\n",
       "\n",
       "[501 rows x 1 columns]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sample3.sort_values(0, ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train3_sample = train_sample3.loc[train_sample3[0] > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20 entries, 29 to 494\n",
      "Data columns (total 1 columns):\n",
      "0    20 non-null float64\n",
      "dtypes: float64(1)\n",
      "memory usage: 320.0 bytes\n"
     ]
    }
   ],
   "source": [
    "#gives us 22 main features\n",
    "#train3_sample\n",
    "train3_sample.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SelectKBest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "select_k = SelectKBest(score_func = f_regression, k=200)\n",
    "k_select = select_k.fit_transform(X, target)\n",
    "df = pd.DataFrame(k_select)\n",
    "#df_boolean = pd.DataFrame(select_k.get_support())\n",
    "#df_boolean = df_boolean[df_boolean[0] != False]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-f3b8f43d9637>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mscore_selectk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselect_k\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscores_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mscore_selectk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtop_7\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscore_selectk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtop_7\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "score_selectk = pd.DataFrame(select_k.scores_)\n",
    "score_selectk.sort_values(0, ascending=False, inplace=True)\n",
    "top_7 = score_selectk.iloc[0:8].T\n",
    "top_7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pandas/indexes/base.py:1434: VisibleDeprecationWarning: boolean index did not match indexed array along dimension 0; dimension is 501 but corresponding boolean dimension is 500\n",
      "  result = getitem(key)\n"
     ]
    }
   ],
   "source": [
    "mask = select_k.get_support()\n",
    "selectkbest = train_set.columns[mask]\n",
    "#pd.DataFrame(selectkbest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "best_features = []\n",
    "column_names = list(train_set.columns.values)\n",
    "\n",
    "for bool, feature in zip(mask, column_names):\n",
    "    if bool:\n",
    "        best_features.append(feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "support_k = pd.DataFrame(best_features)\n",
    "#support_k.iloc[20:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>40</th>\n",
       "      <th>369</th>\n",
       "      <th>285</th>\n",
       "      <th>68</th>\n",
       "      <th>168</th>\n",
       "      <th>41</th>\n",
       "      <th>108</th>\n",
       "      <th>496</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.673211</td>\n",
       "      <td>7.585568</td>\n",
       "      <td>6.718517</td>\n",
       "      <td>6.635876</td>\n",
       "      <td>6.281178</td>\n",
       "      <td>5.975656</td>\n",
       "      <td>5.756965</td>\n",
       "      <td>5.639179</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        40        369       285       68        168       41        108  \\\n",
       "0  9.673211  7.585568  6.718517  6.635876  6.281178  5.975656  5.756965   \n",
       "\n",
       "        496  \n",
       "0  5.639179  "
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#select k percentile with ten percent of the over all features\n",
    "select_k2 = SelectPercentile(score_func = f_regression, percentile=10)\n",
    "k_select2 = select_k2.fit_transform(X, target)\n",
    "select_per = pd.DataFrame(select_k2.scores_).sort_values(0, ascending=False, inplace=False)\n",
    "select_per.iloc[0:8].T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SelectKPercentile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>9.673211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>7.585568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>6.718517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>6.635876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>6.281178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>5.975656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>5.756965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>5.639179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>5.302489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>5.301249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>5.086682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>4.923686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>4.870569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>4.865135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>4.817394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0\n",
       "41   9.673211\n",
       "370  7.585568\n",
       "286  6.718517\n",
       "69   6.635876\n",
       "169  6.281178\n",
       "42   5.975656\n",
       "109  5.756965\n",
       "497  5.639179\n",
       "170  5.302489\n",
       "96   5.301249\n",
       "128  5.086682\n",
       "453  4.923686\n",
       "337  4.870569\n",
       "242  4.865135\n",
       "332  4.817394"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "select_k3 = SelectPercentile(score_func = f_regression, percentile=20)\n",
    "k_select3 = select_k3.fit(train_set, target)\n",
    "select_per2 = pd.DataFrame(select_k3.scores_).sort_values(0, ascending=False, inplace=False)\n",
    "select_per2[0:15]\n",
    "#.iloc[0:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>9.673211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>7.585568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>6.718517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>6.635876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>6.281178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>5.975656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>5.756965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>5.639179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>5.302489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>5.301249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>5.086682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>4.923686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>4.870569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>4.865135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>4.817394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0\n",
       "41   9.673211\n",
       "370  7.585568\n",
       "286  6.718517\n",
       "69   6.635876\n",
       "169  6.281178\n",
       "42   5.975656\n",
       "109  5.756965\n",
       "497  5.639179\n",
       "170  5.302489\n",
       "96   5.301249\n",
       "128  5.086682\n",
       "453  4.923686\n",
       "337  4.870569\n",
       "242  4.865135\n",
       "332  4.817394"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#second\n",
    "select_k4 = SelectPercentile(score_func = f_regression, percentile=30)\n",
    "k_select4 = select_k4.fit_transform(train_set, target)\n",
    "select_per3 = pd.DataFrame(select_k4.scores_).sort_values(0, ascending=False, inplace=False)\n",
    "select_per3.iloc[0:15]\n",
    "#.iloc[0:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mask = top_scores.index \n",
    "\n",
    "last_df = madelon_traindf1.iloc[mask]      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = last_df.drop(['label'], axis=1)\n",
    "target = last_df['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "regressor2 = DecisionTreeRegressor(max_depth=5, min_samples_leaf=3, max_leaf_nodes=20)\n",
    "tree_results2 = regressor2.fit(X, target)\n",
    "score2 = tree_results2.score(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.86531986531986527"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}