scrapping_utils.py

'Class for basic scrapping utility functions'

import logging
from logging.config import fileConfig
from lxml import etree
import os
import re
import requests
from requests.adapters import HTTPAdapter
import shutil

fileConfig('scrappers/logging_config.ini')
logger = logging.getLogger()
DEFAULT_XPATH = ".//text()"
MAX_URL_RETRIES = 25
SAMPLE_URL = "http://www.cbgaindia.org/"
TEXT_DOC_XPATH = "//text()"

class ScrappingUtils(object):
    def __init__(self):
        '''Initializes session for scrapping utils
        '''
        self.session = requests.Session()
        self.session.mount(SAMPLE_URL, HTTPAdapter(MAX_URL_RETRIES))

    def fetch_page(self, url):
        '''Fetches URL and return its textual content, in case of error returns empty text
        '''
        page_text = ''
        try:
            response = self.session.get(url, stream=True)
            page_text = response.text
        except Exception, error_message:
            logger.error("Unable to fetch following URL: %s, error message: %s" % (url, error_message))
        return page_text

    def check_and_create_file_path(self, file_path):
        '''Check if filepath exists else create it
        '''
        try:
            dir_path = os.path.dirname(os.path.realpath(file_path))
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)
        except Exception, error_message:
            logger.error("Unable to create filepath: %s, error message: %s" % (file_path, error_message))  

    def fetch_and_save_file(self, url, file_path):
        '''Fetches URL and save it as file
        '''
        try:
            self.check_and_create_file_path(file_path)
            response = self.session.get(url, stream=True)
            if response.status_code == 200:
                with open(file_path, 'wb') as file_obj:
                    response.raw.decode_content = True
                    shutil.copyfileobj(response.raw, file_obj)
        except Exception, error_message:
            logger.error("Unable to fetch file at following URL: %s, error message: %s" % (url, error_message)) 

    def get_page_dom(self, url):
        '''Fetches page from URL and returns dom tree
        '''
        dom_tree = None
        page_text = self.fetch_page(url)
        if page_text:
            dom_tree = etree.HTML(page_text)
        return dom_tree
    
    def get_links_from_dom(self, dom_tree, xpath):
        '''Fetches links from Page DOM by the xpaths
        '''
        return dom_tree.xpath(xpath)

    def get_links_from_url(self, url, xpath):
        '''Fetches links from URL by the xpaths
        '''
        links = []
        dom_tree = self.get_page_dom(url)
        if dom_tree is not None:
            links = self.get_links_from_dom(dom_tree, xpath)
        return links

    def get_text_from_element(self, element, remove_new_lines=True, xpath=None, join_operator=None):
        '''Retrives text from a etree element in a customizable way
        '''
        if not xpath:
            xpath = DEFAULT_XPATH
        if not join_operator:
            join_operator = ' '
        element_text = join_operator.join(element.xpath(xpath))
        if remove_new_lines:
            element_text = re.sub(r"\s{2,}|\r\n", " ", element_text).strip()
        else:
            element_text = re.sub(r" {2,}|\r\n", " ", element_text).strip()
        return element_text

    def save_file_as_txt(self, page_text, file_path, xpath=TEXT_DOC_XPATH):
        '''Saves HTML page text as TXT file document
        '''
        dom_tree = etree.HTML(page_text)
        doc_text = self.get_text_from_element(dom_tree, remove_new_lines=False, xpath=xpath, join_operator="\n")
        doc_text = re.sub(r"\n{2,}", "\n", doc_text)
        txt_file = open(file_path, "wb")
        txt_file.write(doc_text)
        txt_file.close()