ver-3.1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Results\n",
    "\n",
    "# Both Feature Sets\n",
    "# [349]\tvalidation_0-rmse:0.544184\n",
    "# [160]\tvalidation_0-rmse:0.536579\n",
    "\n",
    "# Single Feature Set\n",
    "# [160]\tvalidation_0-rmse:0.561419\n",
    "\n",
    "# 9th place  max_depth=20, n_estimators=161, learning_rate=0.03, nthread=4, subsample=0.95, colsample_bytree=0.85, seed=4242 )\n",
    "#[160]\tvalidation_0-rmse:0.118262\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 532)\n",
      "(60000, 1)\n",
      "(40000, 532)\n"
     ]
    }
   ],
   "source": [
    "# Iris Data\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import OneClassSVM\n",
    "import sklearn.linear_model as lm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn.cross_validation import StratifiedShuffleSplit\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.cross_validation import cross_val_predict\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "# Data Files\n",
    "\n",
    "# load data\n",
    "df_train_1 = pd.read_csv('./data/kaggle.X1.train.txt', header=None)\n",
    "df_train_2 = pd.read_csv('./data/kaggle.X2.train.txt', header=None)\n",
    "df_train_Y = pd.read_csv('./data/kaggle.Y.train.txt', header=None)\n",
    "df_test_1 = pd.read_csv('./data/kaggle.X1.test.txt', header=None)\n",
    "df_test_2 = pd.read_csv('./data/kaggle.X2.test.txt', header=None)\n",
    "\n",
    "# Combine Train Data \n",
    "df_train_X = pd.concat([df_train_1, df_train_2], axis=1)\n",
    "\n",
    "# Combine Test Data \n",
    "df_test_X = pd.concat([df_test_1, df_test_2], axis=1)\n",
    "\n",
    "\n",
    "# Create Numpy Arrays\n",
    "X_train = df_train_X.values\n",
    "y_train = df_train_Y.values\n",
    "X_test = df_test_X.values\n",
    "\n",
    "# Use Only X1\n",
    "# X_train = df_train_1.values\n",
    "# X_test = df_test_1.values\n",
    "\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Train Model\n",
    "\n",
    "\n",
    "# Split Data\n",
    "X_fit, X_eval, y_fit, y_eval= train_test_split(X_train, y_train, test_size=0.2)\n",
    "\n",
    "\n",
    "# Grid Search \n",
    "clf = xgb.XGBRegressor(silent=False)  \n",
    "\n",
    "\n",
    "# clf_params = {\n",
    "#     'learning_rate' : [0.05,  0.03],\n",
    "#     'n_estimators' : [ 2000, 1000, 500],\n",
    "#     'max_depth' : [10, 7, 5],\n",
    "#     'subsample' : [0.90, 1, 0],\n",
    "#     'colsample_bytree' : [0.85, 1, 0.95],\n",
    "#     'reg_alpha' : [0, 0.7, 1], \n",
    "#     'reg_lambda' : [0, 0.7, 1], \n",
    "#     'seed' : [2314],\n",
    "# }    'min_child_weight' : [7],\n",
    "\n",
    "clf_params = {\n",
    "    'learning_rate' : [ 0.03],\n",
    "    'n_estimators' : [300, 800, 900, 1200 ],\n",
    "    'max_depth' : [ 7, 10, 15, 20 ],\n",
    "    'subsample' : [ 0.90, 0.95],\n",
    "    'colsample_bytree' : [0.9, 0.95],\n",
    "    'reg_alpha' : [ 0.75, 0.85, 0.95, 1], \n",
    "    'seed' : [2314],\n",
    "    'min_child_weight' : [3, 7, 10],\n",
    "    'nthread' : [8],\n",
    "}\n",
    "\n",
    "cv = KFold(4)\n",
    "grid = GridSearchCV(clf, clf_params, cv=cv, n_jobs=-1,  scoring='mean_squared_error', verbose=10 )\n",
    "grid.fit(X_fit, y_fit)\n",
    "\n",
    "\n",
    "print\n",
    "print(\"Best Grid Params:\", grid.best_params_)\n",
    "print(\"Done!\")\n",
    "print(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Score From GRID.best_estimator\n",
    "\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import median_absolute_error\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Print Parameters\n",
    "print(\"Parameters\")\n",
    "print()\n",
    "print(grid.best_estimator_)\n",
    "print(\"-Grid.best_estimator\")\n",
    "print(\"Best Grid Params:\", grid.best_params_)\n",
    "# print(\"Depth: \", depth)\n",
    "# print(\"estimator: \", estimator)\n",
    "# print(\"l_rate: \", l_rate)\n",
    "# print(\"sub_sample: \", sub_sample)\n",
    "# print(\"col_sample: \", col_sample)\n",
    "# print(\"seed: \", seed_val)\n",
    "# print(\"Alpha: \", alpha_val)\n",
    "#print(\"Lambda: \", lambda_val)\n",
    "# print(\"Num Rounds: \", n_round)\n",
    "# print(\"Verbose Val: \", verbose_val)\n",
    "print()\n",
    "print()\n",
    "\n",
    "\n",
    "# Score On Full Train Set\n",
    "# full_train_pred = clf.predict(X_train)\n",
    "full_train_pred = grid.best_estimator_.predict(X_train)\n",
    "print(\" -- Scores -- Full Train -- \")\n",
    "print(\"Explained Variance Score: \", explained_variance_score(y_train, full_train_pred ))\n",
    "print(\"Mean Absolute Error: \", mean_absolute_error(y_train, full_train_pred))\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_train, full_train_pred))\n",
    "print(\"Median Absolute Error: \", median_absolute_error(y_train, full_train_pred))\n",
    "print(\"R2 Score: \", r2_score(y_train, full_train_pred))\n",
    "print()\n",
    "print()\n",
    "\n",
    "# Score On Train Split\n",
    "# split_train_pred = clf.predict(X_fit)\n",
    "split_train_pred = grid.best_estimator_.predict(X_fit)\n",
    "print(\" -- Scores -- Train Split -- \")\n",
    "print(\"Explained Variance Score: \", explained_variance_score(y_fit, split_train_pred ))\n",
    "print(\"Mean Absolute Error: \", mean_absolute_error(y_fit, split_train_pred))\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_fit, split_train_pred))\n",
    "print(\"Median Absolute Error: \", median_absolute_error(y_fit, split_train_pred))\n",
    "print(\"R2 Score: \", r2_score(y_fit, split_train_pred))\n",
    "print()\n",
    "print()\n",
    "\n",
    "# Score On Test Split\n",
    "# split_test_pred = clf.predict(X_eval)\n",
    "split_test_pred = grid.best_estimator_.predict(X_eval)\n",
    "print(\" -- Scores -- Test Split -- \")\n",
    "print(\"Explained Variance Score: \", explained_variance_score(y_eval, split_test_pred ))\n",
    "print(\"Mean Absolute Error: \", mean_absolute_error(y_eval, split_test_pred))\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_eval, split_test_pred))\n",
    "print(\"Median Absolute Error: \", median_absolute_error(y_eval, split_test_pred))\n",
    "print(\"R2 Score: \", r2_score(y_eval, split_test_pred))\n",
    "print()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters\n",
      "\n",
      "XGBRegressor(base_score=0.5, colsample_bylevel=1, colsample_bytree=0.9,\n",
      "       gamma=0, learning_rate=0.03, max_delta_step=0, max_depth=7,\n",
      "       min_child_weight=7, missing=None, n_estimators=300, nthread=8,\n",
      "       objective='reg:linear', reg_alpha=0.75, reg_lambda=1,\n",
      "       scale_pos_weight=1, seed=2314, silent=False, subsample=0.9)\n",
      "-Grid-\n",
      "\n",
      "\n",
      " -- Scores -- Full Train -- \n",
      "Explained Variance Score:  0.683264703432\n",
      "Mean Absolute Error:  0.304814926418\n",
      "Mean Squared Error:  0.220551276693\n",
      "Median Absolute Error:  0.187393305235\n",
      "R2 Score:  0.683264618397\n",
      "\n",
      "\n",
      " -- Scores -- Train Split -- \n",
      "Explained Variance Score:  0.7274447618\n",
      "Mean Absolute Error:  0.288045689526\n",
      "Mean Squared Error:  0.189862939036\n",
      "Median Absolute Error:  0.181682646703\n",
      "R2 Score:  0.727444663004\n",
      "\n",
      "\n",
      " -- Scores -- Test Split -- \n",
      "Explained Variance Score:  0.506192817407\n",
      "Mean Absolute Error:  0.371891873985\n",
      "Mean Squared Error:  0.343304627322\n",
      "Median Absolute Error:  0.213799506426\n",
      "R2 Score:  0.506192777136\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Score From GRID\n",
    "\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import median_absolute_error\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Print Parameters\n",
    "print(\"Parameters\")\n",
    "print()\n",
    "print(grid.best_estimator_)\n",
    "print(\"-Grid-\")\n",
    "#print(\"Best Grid Params:\", grid.best_params_)\n",
    "# print(\"Depth: \", depth)\n",
    "# print(\"estimator: \", estimator)\n",
    "# print(\"l_rate: \", l_rate)\n",
    "# print(\"sub_sample: \", sub_sample)\n",
    "# print(\"col_sample: \", col_sample)\n",
    "# print(\"seed: \", seed_val)\n",
    "# print(\"Alpha: \", alpha_val)\n",
    "#print(\"Lambda: \", lambda_val)\n",
    "# print(\"Num Rounds: \", n_round)\n",
    "# print(\"Verbose Val: \", verbose_val)\n",
    "print()\n",
    "print()\n",
    "\n",
    "\n",
    "# Score On Full Train Set\n",
    "# full_train_pred = clf.predict(X_train)\n",
    "full_train_pred = grid.predict(X_train)\n",
    "print(\" -- Scores -- Full Train -- \")\n",
    "print(\"Explained Variance Score: \", explained_variance_score(y_train, full_train_pred ))\n",
    "print(\"Mean Absolute Error: \", mean_absolute_error(y_train, full_train_pred))\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_train, full_train_pred))\n",
    "print(\"Median Absolute Error: \", median_absolute_error(y_train, full_train_pred))\n",
    "print(\"R2 Score: \", r2_score(y_train, full_train_pred))\n",
    "print()\n",
    "print()\n",
    "\n",
    "# Score On Train Split\n",
    "# split_train_pred = clf.predict(X_fit)\n",
    "split_train_pred = grid.predict(X_fit)\n",
    "print(\" -- Scores -- Train Split -- \")\n",
    "print(\"Explained Variance Score: \", explained_variance_score(y_fit, split_train_pred ))\n",
    "print(\"Mean Absolute Error: \", mean_absolute_error(y_fit, split_train_pred))\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_fit, split_train_pred))\n",
    "print(\"Median Absolute Error: \", median_absolute_error(y_fit, split_train_pred))\n",
    "print(\"R2 Score: \", r2_score(y_fit, split_train_pred))\n",
    "print()\n",
    "print()\n",
    "\n",
    "# Score On Test Split\n",
    "# split_test_pred = clf.predict(X_eval)\n",
    "split_test_pred = grid.predict(X_eval)\n",
    "print(\" -- Scores -- Test Split -- \")\n",
    "print(\"Explained Variance Score: \", explained_variance_score(y_eval, split_test_pred ))\n",
    "print(\"Mean Absolute Error: \", mean_absolute_error(y_eval, split_test_pred))\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_eval, split_test_pred))\n",
    "print(\"Median Absolute Error: \", median_absolute_error(y_eval, split_test_pred))\n",
    "print(\"R2 Score: \", r2_score(y_eval, split_test_pred))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Write To File\n",
    "import csv\n",
    "\n",
    "y_pred = grid.best_estimator_.predict(X_test)\n",
    "\n",
    "write_file = open('predictions_grid_tuesday_1.csv', 'w')\n",
    "write_file.write('ID,Prediction\\n')\n",
    "for x,y in enumerate(y_pred):\n",
    "    write_file.write('{},{} \\n'.format(x+1, y))\n",
    "    \n",
    "                     \n",
    "write_file.close()\n",
    "\n",
    "print(y_pred.shape)\n",
    "\n",
    "print('Completed!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'XGBRegressor' object has no attribute 'decision_function'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-89029c931b8e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mgrid\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecision_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/carlislk/anaconda3/lib/python3.5/site-packages/sklearn/utils/metaestimators.py\u001b[0m in \u001b[0;36m__get__\u001b[1;34m(self, obj, type)\u001b[0m\n\u001b[0;32m     33\u001b[0m             \u001b[1;31m# delegate only on instances, not the classes.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     34\u001b[0m             \u001b[1;31m# this is to allow access to the docstrings.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 35\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     36\u001b[0m         \u001b[1;31m# lambda, but not partial, allows help() to work with update_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'XGBRegressor' object has no attribute 'decision_function'"
     ]
    }
   ],
   "source": [
    "# Score From GRID\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}