india_mart_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a1d20f74-ba7a-46c7-9ae6-7cff790f57b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver  # To initialize and control a browser instance programmatically\n",
    "from selenium.webdriver.chrome.service import Service  # To manage the ChromeDriver executable service for Selenium\n",
    "from selenium.webdriver.chrome.options import Options  # To configure options for the Chrome browser instance (e.g., headless mode)\n",
    "from selenium.webdriver.common.by import By  # To locate elements on a webpage using different strategies (e.g., ID, name, XPath)\n",
    "from selenium.webdriver.support.ui import WebDriverWait  # To pause execution until a specific condition is met (e.g., element visibility)\n",
    "from selenium.webdriver.support import expected_conditions as EC  # To define conditions to wait for (e.g., element clickable, text presence)\n",
    "from selenium.common.exceptions import TimeoutException, NoSuchElementException  # To handle specific Selenium exceptions during interaction\n",
    "import pandas as pd  # For data manipulation and analysis, especially to store scraped data in structured formats like DataFrames\n",
    "import time  # To introduce delays between operations or simulate user-like behavior\n",
    "import requests  # To make HTTP requests (e.g., GET, POST) for retrieving web data or interacting with APIs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3b0f2262-b52c-4870-ac0b-cca83fc62884",
   "metadata": {},
   "outputs": [],
   "source": [
    "service = Service(r'C:\\Users\\data_architect\\Downloads\\chromedriver-win64 (1)\\chromedriver-win64\\chromedriver.exe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7cace48b-a54d-49bd-b73d-8b62156f3a5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page title is: Google\n"
     ]
    }
   ],
   "source": [
    "options = Options()\n",
    "options.add_argument(\"--headless\")\n",
    "\n",
    "service = Service()\n",
    "driver = webdriver.Chrome(service=service, options=options)\n",
    "\n",
    "driver.get('https://www.google.com')\n",
    "print(f\"Page title is: {driver.title}\")\n",
    "\n",
    "driver.quit()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c4a55653-a45a-4f37-b96c-daffad0f9d45",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://dir.indiamart.com/search.mp?ss=korean+flavours&v=4&mcatid=177793&catid=15&prdsrc=1&tags=stype:attr=1|qr_nm=splt-gd|res=RC5|com-cf:nl|ptrs=na|ktp=N0|mc=6529|mtp=G|qry_typ=P|lang=en|wc=2&cs=9275'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d9eb3dff-3746-4480-9d8f-06a840a529a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "646cda7c-a7e5-4725-8835-af028f232b63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "print(r.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b00f6d5-703c-461b-b8d3-91116f5753e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "## working code\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import TimeoutException, NoSuchElementException\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "# Function to scrape data and save specific records in one shot\n",
    "def scrape_and_save_data(url, driver_path, step_size=2):\n",
    "    # Setup Chrome WebDriver\n",
    "    chrome_options = Options()\n",
    "    service = Service(driver_path)\n",
    "    driver = webdriver.Chrome(service=service, options=chrome_options)\n",
    "\n",
    "    # Target URL\n",
    "    driver.get(url)\n",
    "\n",
    "    # Wait for initial elements to load\n",
    "    try:\n",
    "        WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.CLASS_NAME, \"cardlinks\")))\n",
    "        WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, \"span.elps.elps1\")))\n",
    "    except TimeoutException:\n",
    "        print(\"Error: Elements did not load in time.\")\n",
    "        driver.quit()\n",
    "        return\n",
    "\n",
    "    # Lists to store data\n",
    "    product_names = []\n",
    "    product_links = []\n",
    "    seller_names = []\n",
    "    seller_addresses = []\n",
    "\n",
    "    # Function to scrape data\n",
    "    def scrape_data():\n",
    "        try:\n",
    "            products = driver.find_elements(By.CLASS_NAME, \"cardlinks\")\n",
    "            addresses = driver.find_elements(By.CSS_SELECTOR, \"span.elps.elps1\")\n",
    "\n",
    "            for i, product in enumerate(products):\n",
    "                product_name = product.text.strip()\n",
    "                product_link = product.get_attribute('href')\n",
    "\n",
    "                try:\n",
    "                    seller_name = product.find_element(By.XPATH, \".//following-sibling::a\").text.strip()\n",
    "                except NoSuchElementException:\n",
    "                    seller_name = \"N/A\"\n",
    "\n",
    "                seller_address = addresses[i].text.strip() if i < len(addresses) else 'N/A'\n",
    "\n",
    "                product_names.append(product_name)\n",
    "                product_links.append(product_link)\n",
    "                seller_names.append(seller_name)\n",
    "                seller_addresses.append(seller_address)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"Error during scraping: {e}\")\n",
    "\n",
    "    # Pause for manual interaction\n",
    "    print(\"Please log in and click the 'Show More' button manually. Press Enter to continue...\")\n",
    "    input()\n",
    "\n",
    "    # Function to scroll and load more data\n",
    "    def scroll_and_load():\n",
    "        last_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "        while True:\n",
    "            driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "            time.sleep(3)\n",
    "\n",
    "            scrape_data()\n",
    "\n",
    "            new_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "            if new_height == last_height:\n",
    "                print(\"No more data to load.\")\n",
    "                break\n",
    "            last_height = new_height\n",
    "\n",
    "    # Start scrolling and scraping\n",
    "    scroll_and_load()\n",
    "\n",
    "    # Save scraped data to a DataFrame\n",
    "    df = pd.DataFrame({\n",
    "        'Product Name': product_names,\n",
    "        'Product Link': product_links,\n",
    "        'Seller Name': seller_names,\n",
    "        'Seller Address': seller_addresses\n",
    "    })\n",
    "\n",
    "    # Debugging: Print page source and available elements (optional)\n",
    "    print(\"Page Source (first 1000 chars):\", driver.page_source[:1000])\n",
    "    print(\"Products:\", [elem.text for elem in driver.find_elements(By.CLASS_NAME, \"cardlinks\")])\n",
    "    print(\"Addresses:\", [elem.text for elem in driver.find_elements(By.CSS_SELECTOR, \"span.elps.elps1\")])\n",
    "\n",
    "    # Save entire scraped data to Excel\n",
    "    df.to_excel('korean_scraped_data.xlsx', index=False)\n",
    "\n",
    "    # Generate indices for every Nth record\n",
    "    indices = list(range(0, len(df), step_size))  # 0, 2, 4, etc.\n",
    "    specific_records = df.iloc[indices].reset_index(drop=True)\n",
    "\n",
    "    # Extract every Nth record where N is step_size\n",
    "    nth_records = df.iloc[step_size-1::step_size].reset_index(drop=True)\n",
    "\n",
    "    # Save specific records to separate Excel files\n",
    "    specific_records.to_excel(\"korean_products.xlsx\", index=False)\n",
    "    nth_records.to_excel(\"korean_seller.xlsx\", index=False)\n",
    "\n",
    "    # Close the browser\n",
    "    driver.quit()\n",
    "\n",
    "    print(\"Data scraping and saving completed!\")\n",
    "\n",
    "# Example usage of the function\n",
    "scrape_and_save_data(\n",
    "    url=url_korean,  # Replace with the actual URL\n",
    "    driver_path= r'C:\\Users\\data_architect\\Downloads\\chromedriver-win64 (1)\\chromedriver-win64\\chromedriver.exe',\n",
    "    step_size=2\n",
    ")    \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}