scrape_floorsheet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from datetime import datetime\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(driver, date):\n",
    "    \"\"\"\n",
    "    Date in mm/dd/yyyy\n",
    "    \"\"\"\n",
    "    driver.get(\"https://merolagani.com/Floorsheet.aspx\")\n",
    "    date_input = driver.find_element_by_xpath('/html/body/form/div[4]/div[4]/div/div/div[1]/div[4]/input')\n",
    "    search_btn = driver.find_element_by_xpath('/html/body/form/div[4]/div[4]/div/div/div[2]/a[1]')\n",
    "    date_input.send_keys(date)\n",
    "    search_btn.click()\n",
    "    if driver.find_elements_by_xpath(\"//*[contains(text(), 'Could not find floorsheet matching the search criteria')]\"):\n",
    "        print(\"No data found for the given search.\")\n",
    "        print(\"Aborting script ......\")\n",
    "        sys.exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_page_table(driver, table_class):\n",
    "    soup = BeautifulSoup(driver.page_source,'html')\n",
    "    table = soup.find(\"table\", {\"class\":table_class})\n",
    "    tab_data = [[cell.text.replace('\\r', '').replace('\\n', '') for cell in row.find_all([\"th\",\"td\"])]\n",
    "                        for row in table.find_all(\"tr\")]\n",
    "    df = pd.DataFrame(tab_data)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape_data(driver, date):\n",
    "    start_time = datetime.now()\n",
    "    search(driver, date = date)\n",
    "    df = pd.DataFrame()\n",
    "    while True:\n",
    "        page_table_df = get_page_table(driver, table_class=\"table table-bordered table-striped table-hover sortable\")\n",
    "        df = df.append(page_table_df, ignore_index = True)\n",
    "        try:\n",
    "            next_btn = driver.find_element_by_link_text('Next')\n",
    "            driver.execute_script(\"arguments[0].click();\", next_btn)\n",
    "        except NoSuchElementException:\n",
    "            break\n",
    "    print(f\"Time taken to scrape: {datetime.now() - start_time}\")    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_df(df):\n",
    "    new_df = df.drop_duplicates(keep='first') # Dropping Duplicates\n",
    "    new_header = new_df.iloc[0] # grabing the first row for the header\n",
    "    new_df = new_df[1:] # taking the data lower than the header row\n",
    "    new_df.columns = new_header # setting the header row as the df header\n",
    "    new_df.drop([\"#\"], axis=1, inplace=True)\n",
    "    new_df[\"Rate\"] = new_df[\"Rate\"].apply(lambda x:float(x.replace(\",\", \"\"))) # Convert Rate to Float\n",
    "    new_df[\"Amount\"] = new_df[\"Amount\"].apply(lambda x:float(x.replace(\",\", \"\"))) # Convert Amount to Float\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/suyog/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:3: DeprecationWarning: use options instead of chrome_options\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No data found for the given search.\n",
      "Aborting script ......\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/suyog/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3339: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "options = Options()\n",
    "options.headless = True\n",
    "driver = webdriver.Chrome(chrome_options=options) # Start Browser\n",
    "\n",
    "date = datetime.today().strftime('%m/%d/%Y') # Get today's date\n",
    "search(driver, date) # Search the webpage\n",
    "df = scrape_data(driver, date) # Scraping\n",
    "final_df = clean_df(df) # Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = date.replace(\"/\", \"_\")\n",
    "final_df.to_csv(f\"data/{file_name}.csv\", index=False) # Save file"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}