FC Web Scrape Historical NBA Data_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/Users/ronaldholt/dfs_stored_db'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import urllib\n",
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import Select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base_url = \"https://www.fantasycruncher.com/lineup-rewind/draftkings/NBA/DATE\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-12-25\n",
      "2017-12-26\n",
      "2017-12-27\n",
      "2017-12-28\n",
      "2017-12-29\n",
      "2017-12-30\n",
      "2017-12-31\n",
      "2018-01-01\n",
      "2018-01-02\n",
      "2018-01-03\n",
      "2018-01-04\n",
      "2018-01-05\n",
      "2018-01-06\n",
      "2018-01-07\n",
      "2018-01-08\n",
      "2018-01-09\n",
      "2018-01-10\n",
      "2018-01-11\n",
      "2018-01-12\n",
      "2018-01-13\n",
      "2018-01-14\n",
      "2018-01-15\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Chrome('/Users/ronaldholt/Downloads/chromedriver')\n",
    "\n",
    "from datetime import timedelta, date\n",
    "\n",
    "def daterange(start_date, end_date):\n",
    "    for n in range(int ((end_date - start_date).days)):\n",
    "        yield start_date + timedelta(n)\n",
    "\n",
    "#insert date range here\n",
    "start_date = date(2017, 12, 25)\n",
    "end_date = date(2018, 1, 16)\n",
    "\n",
    "for single_date in daterange(start_date, end_date):\n",
    "    print single_date.strftime(\"%Y-%m-%d\") #gives the proper date format\n",
    "    \n",
    "    #new_dates = base_url.replace(\"DATE\", single_date.strftime(\"%Y-%m-%d\")) #inserts into the DATE of the url \n",
    "    driver.get(base_url.replace(\"DATE\", single_date.strftime(\"%Y-%m-%d\")))\n",
    "    #driver.get(new_dates)\n",
    "    \n",
    "### A pop-up will appear and suggest for you to close it ###\n",
    "    close_login = driver.find_element_by_class_name(\"close-login-alert\")\n",
    "    close_login.click()\n",
    "    \n",
    "### we need to gather all players from the drop down...\n",
    "## Use dev tools in chrome; find element name and find value to select... \n",
    "# then run code below\n",
    "    select = Select(driver.find_element_by_name('ff_length'))\n",
    "    select.select_by_value('-1')\n",
    "    \n",
    "### This code \"clicks\" copy player list - this copies player data to the os clipboard\n",
    "    elem1 = driver.find_element_by_link_text(\"copy player list\")\n",
    "    elem1.click()\n",
    "    \n",
    "### Closes pop-up that confirms copied player list\n",
    "    close_copy = driver.find_element_by_class_name(\"sa-confirm-button-container\")\n",
    "    close_copy.click()\n",
    "\n",
    "### Pandas reads clipboard, converts to DF, then exports to CSV... \n",
    "    df2 = pd.read_clipboard()\n",
    "    path = '/Users/ronaldholt/dfs_stored_db/data/'\n",
    "    df2.to_csv(path + single_date.strftime(\"%Y-%m-%d\"))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}