From 9283f37458cb25895ceb9a71fca7dd248ac91c5e Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Fri, 1 Nov 2024 21:56:54 -0700 Subject: [PATCH 01/16] feat: add support for Firefox browser compatibility --- main.py | 23 ++++++++++++++++------- src/utils.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 1d2c77226..41841f5ac 100644 --- a/main.py +++ b/main.py @@ -6,10 +6,12 @@ import click from selenium import webdriver from selenium.webdriver.chrome.service import Service as ChromeService +from selenium.webdriver.firefox.service import Service as FirefoxService from webdriver_manager.chrome import ChromeDriverManager +from webdriver_manager.firefox import GeckoDriverManager from selenium.common.exceptions import WebDriverException from lib_resume_builder_AIHawk import Resume, FacadeManager, ResumeGenerator, StyleManager -from src.utils import chrome_browser_options +from src.utils import chrome_browser_options, firefox_browser_options from src.llm.llm_manager import GPTAnswerer from src.aihawk_authenticator import AIHawkAuthenticator from src.aihawk_bot_facade import AIHawkBotFacade @@ -149,11 +151,16 @@ def file_paths_to_dict(resume_file: Path | None, plain_text_resume_file: Path) - return result -def init_browser() -> webdriver.Chrome: +def init_browser(browser: str = 'chrome') -> webdriver.Chrome | webdriver.Firefox: try: - options = chrome_browser_options() - service = ChromeService(ChromeDriverManager().install()) - return webdriver.Chrome(service=service, options=options) + if browser == 'chrome': + options = chrome_browser_options() + service = ChromeService(ChromeDriverManager().install()) + return webdriver.Chrome(service=service, options=options) + elif browser == 'firefox': + options = firefox_browser_options() + service = FirefoxService(GeckoDriverManager().install()) + return webdriver.Firefox(service=service, options=options) except Exception as e: raise RuntimeError(f"Failed to initialize browser: {str(e)}") @@ -171,7 +178,7 @@ def create_and_run_bot(parameters, llm_api_key): job_application_profile_object = JobApplicationProfile(plain_text_resume) - browser = init_browser() + browser = init_browser(parameters['browser']) login_component = AIHawkAuthenticator(browser) apply_component = AIHawkJobManager(browser) gpt_answerer_component = GPTAnswerer(parameters, llm_api_key) @@ -194,8 +201,9 @@ def create_and_run_bot(parameters, llm_api_key): @click.command() @click.option('--resume', type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path), help="Path to the resume PDF file") +@click.option('--browser', type=click.Choice(['chrome', 'firefox']), default=None, help='Browser to use for the bot') @click.option('--collect', is_flag=True, help="Only collects data job information into data.json file") -def main(collect: False, resume: Path = None): +def main(collect: False, browser: str = 'chrome', resume: Path = None): try: data_folder = Path("data_folder") secrets_file, config_file, plain_text_resume_file, output_folder = FileManager.validate_data_folder(data_folder) @@ -206,6 +214,7 @@ def main(collect: False, resume: Path = None): parameters['uploads'] = FileManager.file_paths_to_dict(resume, plain_text_resume_file) parameters['outputFileDirectory'] = output_folder parameters['collectMode'] = collect + parameters['browser'] = browser create_and_run_bot(parameters, llm_api_key) except ConfigError as ce: diff --git a/src/utils.py b/src/utils.py index a14089d1c..0ed976a44 100644 --- a/src/utils.py +++ b/src/utils.py @@ -21,6 +21,7 @@ logger.add(sys.stderr, level="DEBUG") chromeProfilePath = os.path.join(os.getcwd(), "chrome_profile", "linkedin_profile") +firefoxProfilePath = os.path.join(os.getcwd(), "firefox_profile", "linkedin_profile") def ensure_chrome_profile(): logger.debug(f"Ensuring Chrome profile exists at path: {chromeProfilePath}") @@ -33,6 +34,16 @@ def ensure_chrome_profile(): logger.debug(f"Created Chrome profile directory: {chromeProfilePath}") return chromeProfilePath +def ensure_firefox_profile(): + logger.debug(f"Ensuring Firefox profile exists at path: {firefoxProfilePath}") + profile_dir = os.path.dirname(firefoxProfilePath) + if not os.path.exists(profile_dir): + os.makedirs(profile_dir) + logger.debug(f"Created directory for Firefox profile: {profile_dir}") + if not os.path.exists(firefoxProfilePath): + os.makedirs(firefoxProfilePath) + logger.debug(f"Created Firefox profile directory: {firefoxProfilePath}") + return firefoxProfilePath def is_scrollable(element): scroll_height = element.get_attribute("scrollHeight") @@ -153,6 +164,44 @@ def chrome_browser_options(): return options +def firefox_browser_options(): + logger.debug("Setting Firefox browser options") + ensure_firefox_profile() + options = webdriver.FirefoxOptions() + options.add_argument("--start-maximized") + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--ignore-certificate-errors") + options.add_argument("--disable-extensions") + options.add_argument("--disable-gpu") + options.add_argument("--disable-background-timer-throttling") + options.add_argument("--disable-backgrounding-occluded-windows") + options.add_argument("--disable-translate") + options.add_argument("--disable-popup-blocking") + options.add_argument("--no-first-run") + options.add_argument("--no-default-browser-check") + options.add_argument("--disable-logging") + options.add_argument("--disable-autofill") + options.add_argument("--disable-plugins") + options.add_argument("--disable-animations") + options.add_argument("--disable-cache") + + prefs = { + "permissions.default.image": 2, + "permissions.default.stylesheet": 2, + } + for key, value in prefs.items(): + options.set_preference(key, value) + + if len(firefoxProfilePath) > 0: # You'll need to define firefoxProfilePath similar to chromeProfilePath + profile = webdriver.FirefoxProfile(firefoxProfilePath) + options.profile = profile + logger.debug(f"Using Firefox profile directory: {firefoxProfilePath}") + else: + options.set_preference("browser.privatebrowsing.autostart", True) + logger.debug("Using Firefox in private browsing mode") + + return options def printred(text): red = "\033[91m" From 8eb24acf8564f9ed8cd36b9722f74d0b380e8c91 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Sun, 3 Nov 2024 21:40:53 -0800 Subject: [PATCH 02/16] refactor: implement factory pattern for browsers --- data_folder/config.yaml | 2 + main.py | 31 ++++----- src/utils.py | 107 ------------------------------ src/webdrivers/base_browser.py | 87 ++++++++++++++++++++++++ src/webdrivers/browser_factory.py | 49 ++++++++++++++ src/webdrivers/chrome.py | 59 ++++++++++++++++ src/webdrivers/firefox.py | 57 ++++++++++++++++ 7 files changed, 266 insertions(+), 126 deletions(-) create mode 100644 src/webdrivers/base_browser.py create mode 100644 src/webdrivers/browser_factory.py create mode 100644 src/webdrivers/chrome.py create mode 100644 src/webdrivers/firefox.py diff --git a/data_folder/config.yaml b/data_folder/config.yaml index 25616cd4d..4fff37df6 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -51,3 +51,5 @@ job_applicants_threshold: llm_model_type: openai llm_model: 'gpt-4o-mini' # llm_api_url: 'https://api.pawan.krd/cosmosrp/v1' + +# browser: firefox \ No newline at end of file diff --git a/main.py b/main.py index 41841f5ac..e8d21eec1 100644 --- a/main.py +++ b/main.py @@ -5,13 +5,10 @@ import yaml import click from selenium import webdriver -from selenium.webdriver.chrome.service import Service as ChromeService -from selenium.webdriver.firefox.service import Service as FirefoxService -from webdriver_manager.chrome import ChromeDriverManager -from webdriver_manager.firefox import GeckoDriverManager from selenium.common.exceptions import WebDriverException from lib_resume_builder_AIHawk import Resume, FacadeManager, ResumeGenerator, StyleManager -from src.utils import chrome_browser_options, firefox_browser_options +from src.webdrivers.base_browser import BrowserType +from src.webdrivers.browser_factory import BrowserFactory from src.llm.llm_manager import GPTAnswerer from src.aihawk_authenticator import AIHawkAuthenticator from src.aihawk_bot_facade import AIHawkBotFacade @@ -151,16 +148,9 @@ def file_paths_to_dict(resume_file: Path | None, plain_text_resume_file: Path) - return result -def init_browser(browser: str = 'chrome') -> webdriver.Chrome | webdriver.Firefox: +def init_browser(browser_type: BrowserType = BrowserType.CHROME) -> webdriver.Chrome | webdriver.Firefox: try: - if browser == 'chrome': - options = chrome_browser_options() - service = ChromeService(ChromeDriverManager().install()) - return webdriver.Chrome(service=service, options=options) - elif browser == 'firefox': - options = firefox_browser_options() - service = FirefoxService(GeckoDriverManager().install()) - return webdriver.Firefox(service=service, options=options) + return BrowserFactory.get_driver(browser_type) except Exception as e: raise RuntimeError(f"Failed to initialize browser: {str(e)}") @@ -178,7 +168,7 @@ def create_and_run_bot(parameters, llm_api_key): job_application_profile_object = JobApplicationProfile(plain_text_resume) - browser = init_browser(parameters['browser']) + browser = init_browser(BrowserFactory.get_browser_type()) login_component = AIHawkAuthenticator(browser) apply_component = AIHawkJobManager(browser) gpt_answerer_component = GPTAnswerer(parameters, llm_api_key) @@ -201,9 +191,8 @@ def create_and_run_bot(parameters, llm_api_key): @click.command() @click.option('--resume', type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path), help="Path to the resume PDF file") -@click.option('--browser', type=click.Choice(['chrome', 'firefox']), default=None, help='Browser to use for the bot') @click.option('--collect', is_flag=True, help="Only collects data job information into data.json file") -def main(collect: False, browser: str = 'chrome', resume: Path = None): +def main(collect: False, resume: Path = None): try: data_folder = Path("data_folder") secrets_file, config_file, plain_text_resume_file, output_folder = FileManager.validate_data_folder(data_folder) @@ -214,8 +203,12 @@ def main(collect: False, browser: str = 'chrome', resume: Path = None): parameters['uploads'] = FileManager.file_paths_to_dict(resume, plain_text_resume_file) parameters['outputFileDirectory'] = output_folder parameters['collectMode'] = collect - parameters['browser'] = browser - + + BrowserFactory.set_browser_type( + BrowserType.CHROME if parameters.get('browser') is None or parameters['browser'].lower() == 'chrome' + else BrowserType.FIREFOX + ) + create_and_run_bot(parameters, llm_api_key) except ConfigError as ce: logger.error(f"Configuration error: {str(ce)}") diff --git a/src/utils.py b/src/utils.py index 0ed976a44..3ad62d8b0 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,10 +1,8 @@ -import logging import os import random import sys import time -from selenium import webdriver from loguru import logger from app_config import MINIMUM_LOG_LEVEL @@ -23,28 +21,6 @@ chromeProfilePath = os.path.join(os.getcwd(), "chrome_profile", "linkedin_profile") firefoxProfilePath = os.path.join(os.getcwd(), "firefox_profile", "linkedin_profile") -def ensure_chrome_profile(): - logger.debug(f"Ensuring Chrome profile exists at path: {chromeProfilePath}") - profile_dir = os.path.dirname(chromeProfilePath) - if not os.path.exists(profile_dir): - os.makedirs(profile_dir) - logger.debug(f"Created directory for Chrome profile: {profile_dir}") - if not os.path.exists(chromeProfilePath): - os.makedirs(chromeProfilePath) - logger.debug(f"Created Chrome profile directory: {chromeProfilePath}") - return chromeProfilePath - -def ensure_firefox_profile(): - logger.debug(f"Ensuring Firefox profile exists at path: {firefoxProfilePath}") - profile_dir = os.path.dirname(firefoxProfilePath) - if not os.path.exists(profile_dir): - os.makedirs(profile_dir) - logger.debug(f"Created directory for Firefox profile: {profile_dir}") - if not os.path.exists(firefoxProfilePath): - os.makedirs(firefoxProfilePath) - logger.debug(f"Created Firefox profile directory: {firefoxProfilePath}") - return firefoxProfilePath - def is_scrollable(element): scroll_height = element.get_attribute("scrollHeight") client_height = element.get_attribute("clientHeight") @@ -52,7 +28,6 @@ def is_scrollable(element): logger.debug(f"Element scrollable check: scrollHeight={scroll_height}, clientHeight={client_height}, scrollable={scrollable}") return scrollable - def scroll_slow(driver, scrollable_element, start=0, end=3600, step=300, reverse=False): logger.debug(f"Starting slow scroll: start={start}, end={end}, step={step}, reverse={reverse}") @@ -121,88 +96,6 @@ def scroll_slow(driver, scrollable_element, start=0, end=3600, step=300, reverse except Exception as e: logger.error(f"Exception occurred during scrolling: {e}") - -def chrome_browser_options(): - logger.debug("Setting Chrome browser options") - ensure_chrome_profile() - options = webdriver.ChromeOptions() - options.add_argument("--start-maximized") - options.add_argument("--no-sandbox") - options.add_argument("--disable-dev-shm-usage") - options.add_argument("--ignore-certificate-errors") - options.add_argument("--disable-extensions") - options.add_argument("--disable-gpu") - options.add_argument("window-size=1200x800") - options.add_argument("--disable-background-timer-throttling") - options.add_argument("--disable-backgrounding-occluded-windows") - options.add_argument("--disable-translate") - options.add_argument("--disable-popup-blocking") - options.add_argument("--no-first-run") - options.add_argument("--no-default-browser-check") - options.add_argument("--disable-logging") - options.add_argument("--disable-autofill") - options.add_argument("--disable-plugins") - options.add_argument("--disable-animations") - options.add_argument("--disable-cache") - options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"]) - - prefs = { - "profile.default_content_setting_values.images": 2, - "profile.managed_default_content_settings.stylesheets": 2, - } - options.add_experimental_option("prefs", prefs) - - if len(chromeProfilePath) > 0: - initial_path = os.path.dirname(chromeProfilePath) - profile_dir = os.path.basename(chromeProfilePath) - options.add_argument('--user-data-dir=' + initial_path) - options.add_argument("--profile-directory=" + profile_dir) - logger.debug(f"Using Chrome profile directory: {chromeProfilePath}") - else: - options.add_argument("--incognito") - logger.debug("Using Chrome in incognito mode") - - return options - -def firefox_browser_options(): - logger.debug("Setting Firefox browser options") - ensure_firefox_profile() - options = webdriver.FirefoxOptions() - options.add_argument("--start-maximized") - options.add_argument("--no-sandbox") - options.add_argument("--disable-dev-shm-usage") - options.add_argument("--ignore-certificate-errors") - options.add_argument("--disable-extensions") - options.add_argument("--disable-gpu") - options.add_argument("--disable-background-timer-throttling") - options.add_argument("--disable-backgrounding-occluded-windows") - options.add_argument("--disable-translate") - options.add_argument("--disable-popup-blocking") - options.add_argument("--no-first-run") - options.add_argument("--no-default-browser-check") - options.add_argument("--disable-logging") - options.add_argument("--disable-autofill") - options.add_argument("--disable-plugins") - options.add_argument("--disable-animations") - options.add_argument("--disable-cache") - - prefs = { - "permissions.default.image": 2, - "permissions.default.stylesheet": 2, - } - for key, value in prefs.items(): - options.set_preference(key, value) - - if len(firefoxProfilePath) > 0: # You'll need to define firefoxProfilePath similar to chromeProfilePath - profile = webdriver.FirefoxProfile(firefoxProfilePath) - options.profile = profile - logger.debug(f"Using Firefox profile directory: {firefoxProfilePath}") - else: - options.set_preference("browser.privatebrowsing.autostart", True) - logger.debug("Using Firefox in private browsing mode") - - return options - def printred(text): red = "\033[91m" reset = "\033[0m" diff --git a/src/webdrivers/base_browser.py b/src/webdrivers/base_browser.py new file mode 100644 index 000000000..c1e453e28 --- /dev/null +++ b/src/webdrivers/base_browser.py @@ -0,0 +1,87 @@ +from enum import Enum, auto +import os +import sys + +from abc import ABC, abstractmethod +from loguru import logger + +from app_config import MINIMUM_LOG_LEVEL + +log_file = "app_log.log" + + +if MINIMUM_LOG_LEVEL in ["DEBUG", "TRACE", "INFO", "WARNING", "ERROR", "CRITICAL"]: + logger.remove() + logger.add(sys.stderr, level=MINIMUM_LOG_LEVEL) +else: + logger.warning(f"Invalid log level: {MINIMUM_LOG_LEVEL}. Defaulting to DEBUG.") + logger.remove() + logger.add(sys.stderr, level="DEBUG") + + +class BrowserType(Enum): + """Enum for supported browser types""" + CHROME = auto() + FIREFOX = auto() + +class BrowserProfile: + """Manages browser profile creation and configuration""" + def __init__(self, browser_type: BrowserType): + self.browser_type: BrowserType = browser_type + self.profile_path = os.path.join( + os.getcwd(), + f"{self.browser_type}_profile", + "linkedin_profile" + ) + + def ensure_profile_exists(self) -> str: + """ + Ensures the browser profile directory exists + Returns: Path to the profile directory + """ + logger.debug(f"Ensuring {self.browser_type} profile exists at path: {self.profile_path}") + profile_dir = os.path.dirname(self.profile_path) + + if not os.path.exists(profile_dir): + os.makedirs(profile_dir) + logger.debug(f"Created directory for {self.browser_type} profile: {profile_dir}") + + if not os.path.exists(self.profile_path): + os.makedirs(self.profile_path) + logger.debug(f"Created {self.browser_type} profile directory: {self.profile_path}") + + return self.profile_path + +class Browser(ABC): + """Abstract base class for browser implementations""" + def __init__(self): + self.profile = BrowserProfile(self.browser_type) + + @property + def browser_type(self) -> str: + """Return the browser type identifier""" + return self.__class__.browser_type + + @abstractmethod + def create_options(self): + """Create and return browser-specific options""" + + @abstractmethod + def create_service(self): + """Create and return browser-specific service""" + + def create_driver(self): + """Create and return browser-specific WebDriver instance""" + try: + options = self.create_options() + service = self.create_service() + driver = self._create_driver_instance(service, options) + logger.debug(f"{self.browser_type} WebDriver instance created successfully") + return driver + except Exception as e: + logger.error(f"Failed to create {self.browser_type} WebDriver: {e}") + raise RuntimeError(f"Failed to initialize {self.browser_type} browser: {str(e)}") + + @abstractmethod + def _create_driver_instance(self, service, options): + """Create the specific driver instance""" diff --git a/src/webdrivers/browser_factory.py b/src/webdrivers/browser_factory.py new file mode 100644 index 000000000..fa4507d95 --- /dev/null +++ b/src/webdrivers/browser_factory.py @@ -0,0 +1,49 @@ +from typing import Union + +from selenium import webdriver +from loguru import logger + +from src.webdrivers.base_browser import BrowserType +from src.webdrivers.chrome import Chrome +from src.webdrivers.firefox import Firefox + + +class BrowserFactory: + """Factory class for creating browser instances""" + _browser_type: BrowserType = BrowserType.CHROME # Default browser type + + _browsers = { + BrowserType.CHROME: Chrome, + BrowserType.FIREFOX: Firefox + } + + @classmethod + def get_browser_type(cls) -> BrowserType: + """Get current browser type""" + return cls._browser_type + + @classmethod + def set_browser_type(cls, browser_type: BrowserType) -> None: + """Set browser type""" + if browser_type not in cls._browsers: + raise ValueError(f"Unsupported browser type: {browser_type}") + cls._browser_type = browser_type + logger.debug(f"Browser type set to: {browser_type}") + + @classmethod + def get_driver(cls, browser_type: BrowserType) -> Union[webdriver.Chrome, webdriver.Firefox]: + """ + Create and return a WebDriver instance for the specified browser type + Args: + browser_type: BrowserType enum value + Returns: + WebDriver instance + Raises: + RuntimeError: If browser initialization fails + """ + browser_class = cls._browsers.get(browser_type) + if not browser_class: + raise ValueError(f"Unsupported browser type: {browser_type}") + + browser = browser_class() + return browser.create_driver() diff --git a/src/webdrivers/chrome.py b/src/webdrivers/chrome.py new file mode 100644 index 000000000..5b1b17d1f --- /dev/null +++ b/src/webdrivers/chrome.py @@ -0,0 +1,59 @@ +import os +from loguru import logger +from selenium import webdriver +from selenium.webdriver.chrome.service import Service as ChromeService +from webdriver_manager.chrome import ChromeDriverManager + +from src.webdrivers.base_browser import Browser, BrowserType + +chromeProfilePath = os.path.join(os.getcwd(), "chrome_profile", "linkedin_profile") + +class Chrome(Browser): + """Chrome browser implementation""" + browser_type: BrowserType = BrowserType.CHROME + + def create_options(self) -> webdriver.ChromeOptions: + """Create Chrome-specific options""" + self.profile.ensure_profile_exists() + options = webdriver.ChromeOptions() + + chrome_arguments = [ + "--start-maximized", "--no-sandbox", "--disable-dev-shm-usage", + "--ignore-certificate-errors", "--disable-extensions", "--disable-gpu", + "window-size=1200x800", "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", "--disable-translate", + "--disable-popup-blocking", "--no-first-run", "--no-default-browser-check", + "--disable-logging", "--disable-autofill", "--disable-plugins", + "--disable-animations", "--disable-cache" + ] + + for arg in chrome_arguments: + options.add_argument(arg) + + options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"]) + + prefs = { + "profile.default_content_setting_values.images": 2, + "profile.managed_default_content_settings.stylesheets": 2, + } + options.add_experimental_option("prefs", prefs) + + if self.profile.profile_path: + initial_path = os.path.dirname(self.profile.profile_path) + profile_dir = os.path.basename(self.profile.profile_path) + options.add_argument('--user-data-dir=' + initial_path) + options.add_argument("--profile-directory=" + profile_dir) + logger.debug(f"Using Chrome profile directory: {self.profile.profile_path}") + else: + options.add_argument("--incognito") + logger.debug("Using Chrome in incognito mode") + + return options + + def create_service(self) -> ChromeService: + """Create Chrome-specific service""" + return ChromeService(ChromeDriverManager().install()) + + def _create_driver_instance(self, service, options): + """Create Chrome WebDriver instance""" + return webdriver.Chrome(service=service, options=options) diff --git a/src/webdrivers/firefox.py b/src/webdrivers/firefox.py new file mode 100644 index 000000000..9660e049d --- /dev/null +++ b/src/webdrivers/firefox.py @@ -0,0 +1,57 @@ +import os + +from loguru import logger +from selenium import webdriver +from selenium.webdriver.firefox.service import Service as FirefoxService +from webdriver_manager.firefox import GeckoDriverManager + +from src.webdrivers.base_browser import Browser, BrowserType + +firefoxProfilePath = os.path.join(os.getcwd(), "firefox_profile", "linkedin_profile") + +class Firefox(Browser): + """Firefox browser implementation""" + browser_type: BrowserType = BrowserType.FIREFOX + + def create_options(self) -> webdriver.FirefoxOptions: + """Create Firefox-specific options""" + self.profile.ensure_profile_exists() + options = webdriver.FirefoxOptions() + + firefox_arguments = [ + "--start-maximized", "--no-sandbox", "--disable-dev-shm-usage", + "--ignore-certificate-errors", "--disable-extensions", "--disable-gpu", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", "--disable-translate", + "--disable-popup-blocking", "--no-first-run", "--no-default-browser-check", + "--disable-logging", "--disable-autofill", "--disable-plugins", + "--disable-animations", "--disable-cache" + ] + + for arg in firefox_arguments: + options.add_argument(arg) + + prefs = { + "permissions.default.image": 2, + "permissions.default.stylesheet": 2, + } + for key, value in prefs.items(): + options.set_preference(key, value) + + if self.profile.profile_path: + profile = webdriver.FirefoxProfile(self.profile.profile_path) + options.profile = profile + logger.debug(f"Using Firefox profile directory: {self.profile.profile_path}") + else: + options.set_preference("browser.privatebrowsing.autostart", True) + logger.debug("Using Firefox in private browsing mode") + + return options + + def create_service(self) -> FirefoxService: + """Create Firefox-specific service""" + return FirefoxService(GeckoDriverManager().install()) + + def _create_driver_instance(self, service, options): + """Create Firefox WebDriver instance""" + return webdriver.Firefox(service=service, options=options) From 081ae47c225b1d58c5ad63b545b7e7f9176feab2 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Sun, 3 Nov 2024 22:25:07 -0800 Subject: [PATCH 03/16] refactor: remove duplicates --- src/utils.py | 3 --- src/webdrivers/base_browser.py | 14 -------------- 2 files changed, 17 deletions(-) diff --git a/src/utils.py b/src/utils.py index 3ad62d8b0..8b9b26450 100644 --- a/src/utils.py +++ b/src/utils.py @@ -18,9 +18,6 @@ logger.remove() logger.add(sys.stderr, level="DEBUG") -chromeProfilePath = os.path.join(os.getcwd(), "chrome_profile", "linkedin_profile") -firefoxProfilePath = os.path.join(os.getcwd(), "firefox_profile", "linkedin_profile") - def is_scrollable(element): scroll_height = element.get_attribute("scrollHeight") client_height = element.get_attribute("clientHeight") diff --git a/src/webdrivers/base_browser.py b/src/webdrivers/base_browser.py index c1e453e28..c6b9bceab 100644 --- a/src/webdrivers/base_browser.py +++ b/src/webdrivers/base_browser.py @@ -1,23 +1,9 @@ from enum import Enum, auto import os -import sys from abc import ABC, abstractmethod from loguru import logger -from app_config import MINIMUM_LOG_LEVEL - -log_file = "app_log.log" - - -if MINIMUM_LOG_LEVEL in ["DEBUG", "TRACE", "INFO", "WARNING", "ERROR", "CRITICAL"]: - logger.remove() - logger.add(sys.stderr, level=MINIMUM_LOG_LEVEL) -else: - logger.warning(f"Invalid log level: {MINIMUM_LOG_LEVEL}. Defaulting to DEBUG.") - logger.remove() - logger.add(sys.stderr, level="DEBUG") - class BrowserType(Enum): """Enum for supported browser types""" From 227875f950023cc11536d1286bd9f334cc407c4c Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Sun, 3 Nov 2024 22:38:46 -0800 Subject: [PATCH 04/16] refactor: remote duplicates and change the browser type names --- src/utils.py | 1 - src/webdrivers/base_browser.py | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/utils.py b/src/utils.py index 8b9b26450..f37b33d05 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,4 +1,3 @@ -import os import random import sys import time diff --git a/src/webdrivers/base_browser.py b/src/webdrivers/base_browser.py index c6b9bceab..32dc9beaa 100644 --- a/src/webdrivers/base_browser.py +++ b/src/webdrivers/base_browser.py @@ -1,21 +1,21 @@ -from enum import Enum, auto import os from abc import ABC, abstractmethod +from enum import Enum from loguru import logger class BrowserType(Enum): """Enum for supported browser types""" - CHROME = auto() - FIREFOX = auto() - + CHROME = 'chrome' + FIREFOX = 'firefox' + class BrowserProfile: """Manages browser profile creation and configuration""" def __init__(self, browser_type: BrowserType): - self.browser_type: BrowserType = browser_type + self.browser_type: BrowserType = browser_type.name.lower() self.profile_path = os.path.join( - os.getcwd(), + os.getcwd(), f"{self.browser_type}_profile", "linkedin_profile" ) From 0ec4325c77f30da30741deb7609ade4f965ae1ed Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Fri, 8 Nov 2024 00:56:51 -0800 Subject: [PATCH 05/16] remove duplicates --- src/utils/chrome_utils.py | 60 --------------------------------------- tests/test_utils.py | 18 ------------ 2 files changed, 78 deletions(-) delete mode 100644 src/utils/chrome_utils.py diff --git a/src/utils/chrome_utils.py b/src/utils/chrome_utils.py deleted file mode 100644 index 3d3a84ac3..000000000 --- a/src/utils/chrome_utils.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -from selenium import webdriver -from src.logging import logger - -chromeProfilePath = os.path.join(os.getcwd(), "chrome_profile", "linkedin_profile") - -def ensure_chrome_profile(): - logger.debug(f"Ensuring Chrome profile exists at path: {chromeProfilePath}") - profile_dir = os.path.dirname(chromeProfilePath) - if not os.path.exists(profile_dir): - os.makedirs(profile_dir) - logger.debug(f"Created directory for Chrome profile: {profile_dir}") - if not os.path.exists(chromeProfilePath): - os.makedirs(chromeProfilePath) - logger.debug(f"Created Chrome profile directory: {chromeProfilePath}") - return chromeProfilePath - -def chrome_browser_options(): - logger.debug("Setting Chrome browser options") - ensure_chrome_profile() - options = webdriver.ChromeOptions() - options.add_argument("--start-maximized") - options.add_argument("--no-sandbox") - options.add_argument("--disable-dev-shm-usage") - options.add_argument("--ignore-certificate-errors") - options.add_argument("--disable-extensions") - options.add_argument("--disable-gpu") - options.add_argument("window-size=1200x800") - options.add_argument("--disable-background-timer-throttling") - options.add_argument("--disable-backgrounding-occluded-windows") - options.add_argument("--disable-translate") - options.add_argument("--disable-popup-blocking") - options.add_argument("--no-first-run") - options.add_argument("--no-default-browser-check") - options.add_argument("--disable-logging") - options.add_argument("--disable-autofill") - options.add_argument("--disable-plugins") - options.add_argument("--disable-animations") - options.add_argument("--disable-cache") - options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"]) - - prefs = { - "profile.default_content_setting_values.images": 2, - "profile.managed_default_content_settings.stylesheets": 2, - } - options.add_experimental_option("prefs", prefs) - - if len(chromeProfilePath) > 0: - initial_path = os.path.dirname(chromeProfilePath) - profile_dir = os.path.basename(chromeProfilePath) - options.add_argument('--user-data-dir=' + initial_path) - options.add_argument("--profile-directory=" + profile_dir) - logger.debug(f"Using Chrome profile directory: {chromeProfilePath}") - else: - options.add_argument("--incognito") - logger.debug("Using Chrome in incognito mode") - - return options - - diff --git a/tests/test_utils.py b/tests/test_utils.py index 2ca828b44..b97a3868c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,6 @@ from unittest import mock from selenium.webdriver.remote.webelement import WebElement from src.utils.browser_utils import is_scrollable, scroll_slow -from src.utils.chrome_utils import chrome_browser_options, ensure_chrome_profile # Mocking logging to avoid actual file writing @pytest.fixture(autouse=True) @@ -67,20 +66,3 @@ def test_scroll_slow_element_not_scrollable(mocker): # Ensure it detected non-scrollable element mock_driver.execute_script.assert_not_called() - -# Test chrome_browser_options function -def test_chrome_browser_options(mocker): - mocker.patch("src.utils.chrome_utils.ensure_chrome_profile") - mocker.patch("os.path.dirname", return_value="/mocked/path") - mocker.patch("os.path.basename", return_value="profile_directory") - - mock_options = mocker.Mock() - - mocker.patch("selenium.webdriver.ChromeOptions", return_value=mock_options) - - # Call the function - options = chrome_browser_options() - - # Ensure options were set - assert mock_options.add_argument.called - assert options == mock_options From e400a257e7a98b2bdffd790b44f53d62b73100c7 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:10:01 -0800 Subject: [PATCH 06/16] fix: package path --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 42ad6776f..26c60d1dd 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,7 @@ from selenium.common.exceptions import WebDriverException from lib_resume_builder_AIHawk import Resume, FacadeManager, ResumeGenerator, StyleManager from typing import Optional -from src.webdrivers.base_browser.chrome_utils import BrowserType +from src.webdrivers.base_browser import BrowserType from src.webdrivers.browser_factory import BrowserFactory from src.job_application_profile import JobApplicationProfile From 65dc6b4ef273ae42e1e6f117eaabc6859a54d241 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:21:09 -0800 Subject: [PATCH 07/16] refactor: move the browser config to config.py instead --- config.py | 4 +++- data_folder/config.yaml | 2 +- main.py | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index c1cbfcefa..29574e11e 100644 --- a/config.py +++ b/config.py @@ -1,6 +1,7 @@ # In this file, you can set the configurations of the app. from constants import DEBUG +from src.webdrivers.base_browser import BrowserType #config related to logging must have prefix LOG_ LOG_LEVEL = DEBUG @@ -11,4 +12,5 @@ MINIMUM_WAIT_TIME_IN_SECONDS = 60 JOB_APPLICATIONS_DIR = "job_applications" -JOB_SUITABILITY_SCORE = 7 \ No newline at end of file +JOB_SUITABILITY_SCORE = 7 +BROWSER_TYPE_CONFIG = BrowserType.CHROME diff --git a/data_folder/config.yaml b/data_folder/config.yaml index 1948490e0..81c05aa6e 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -54,4 +54,4 @@ llm_model_type: openai llm_model: 'gpt-4o-mini' # llm_api_url: 'https://api.pawan.krd/cosmosrp/v1' -# browser: firefox \ No newline at end of file +browser: firefox \ No newline at end of file diff --git a/main.py b/main.py index 26c60d1dd..24347e39b 100644 --- a/main.py +++ b/main.py @@ -8,9 +8,10 @@ from selenium.common.exceptions import WebDriverException from lib_resume_builder_AIHawk import Resume, FacadeManager, ResumeGenerator, StyleManager from typing import Optional + +from config import BROWSER_TYPE_CONFIG from src.webdrivers.base_browser import BrowserType from src.webdrivers.browser_factory import BrowserFactory - from src.job_application_profile import JobApplicationProfile from src.logging import logger @@ -211,9 +212,9 @@ def main(collect: bool = False, resume: Optional[Path] = None): parameters['uploads'] = FileManager.file_paths_to_dict(resume, plain_text_resume_file) parameters['outputFileDirectory'] = output_folder parameters['collectMode'] = collect - + # check if the config is not set as well BrowserFactory.set_browser_type( - BrowserType.CHROME if parameters.get('browser') is None or parameters['browser'].lower() == 'chrome' + BrowserType.CHROME if BROWSER_TYPE_CONFIG is None or BROWSER_TYPE_CONFIG == BrowserType.CHROME else BrowserType.FIREFOX ) From 7a4f4c73ef914fe564f8e06d6eda6655d2b2bd1c Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Fri, 8 Nov 2024 15:24:49 -0800 Subject: [PATCH 08/16] update unit test cases and remove duplicates --- data_folder/config.yaml | 2 -- main.py | 6 +----- src/webdrivers/browser_factory.py | 6 ++++++ tests/test_utils.py | 36 +++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/data_folder/config.yaml b/data_folder/config.yaml index 81c05aa6e..f41a0c6d3 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -53,5 +53,3 @@ job_applicants_threshold: llm_model_type: openai llm_model: 'gpt-4o-mini' # llm_api_url: 'https://api.pawan.krd/cosmosrp/v1' - -browser: firefox \ No newline at end of file diff --git a/main.py b/main.py index 24347e39b..112c9a8ce 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,6 @@ from lib_resume_builder_AIHawk import Resume, FacadeManager, ResumeGenerator, StyleManager from typing import Optional -from config import BROWSER_TYPE_CONFIG from src.webdrivers.base_browser import BrowserType from src.webdrivers.browser_factory import BrowserFactory from src.job_application_profile import JobApplicationProfile @@ -213,10 +212,7 @@ def main(collect: bool = False, resume: Optional[Path] = None): parameters['outputFileDirectory'] = output_folder parameters['collectMode'] = collect # check if the config is not set as well - BrowserFactory.set_browser_type( - BrowserType.CHROME if BROWSER_TYPE_CONFIG is None or BROWSER_TYPE_CONFIG == BrowserType.CHROME - else BrowserType.FIREFOX - ) + BrowserFactory.init_browser_type() create_and_run_bot(parameters, llm_api_key) except ConfigError as ce: diff --git a/src/webdrivers/browser_factory.py b/src/webdrivers/browser_factory.py index fa4507d95..29ee78ae5 100644 --- a/src/webdrivers/browser_factory.py +++ b/src/webdrivers/browser_factory.py @@ -3,6 +3,7 @@ from selenium import webdriver from loguru import logger +from config import BROWSER_TYPE_CONFIG from src.webdrivers.base_browser import BrowserType from src.webdrivers.chrome import Chrome from src.webdrivers.firefox import Firefox @@ -30,6 +31,11 @@ def set_browser_type(cls, browser_type: BrowserType) -> None: cls._browser_type = browser_type logger.debug(f"Browser type set to: {browser_type}") + @classmethod + def init_browser_type(cls) -> None: + """Initialize browser type from the config""" + cls.set_browser_type(BROWSER_TYPE_CONFIG) + @classmethod def get_driver(cls, browser_type: BrowserType) -> Union[webdriver.Chrome, webdriver.Firefox]: """ diff --git a/tests/test_utils.py b/tests/test_utils.py index b97a3868c..2aa2986c7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,8 @@ from unittest import mock from selenium.webdriver.remote.webelement import WebElement from src.utils.browser_utils import is_scrollable, scroll_slow +from src.webdrivers.chrome import Chrome +from src.webdrivers.firefox import Firefox # Mocking logging to avoid actual file writing @pytest.fixture(autouse=True) @@ -66,3 +68,37 @@ def test_scroll_slow_element_not_scrollable(mocker): # Ensure it detected non-scrollable element mock_driver.execute_script.assert_not_called() + +# Test chrome_browser_options function +def test_chrome_browser_options(mocker): + mocker.patch("src.utils.chrome_utils.ensure_chrome_profile") + mocker.patch("os.path.dirname", return_value="/mocked/path") + mocker.patch("os.path.basename", return_value="profile_directory") + + mock_options = mocker.Mock() + + mocker.patch("selenium.webdriver.ChromeOptions", return_value=mock_options) + + # Call the function + options = Chrome.creaet_options() + + # Ensure options were set + assert mock_options.add_argument.called + assert options == mock_options + +# Test firefox_browser_options function +def test_firefox_browser_options(mocker): + mocker.patch("src.utils.chrome_utils.ensure_chrome_profile") + mocker.patch("os.path.dirname", return_value="/mocked/path") + mocker.patch("os.path.basename", return_value="profile_directory") + + mock_options = mocker.Mock() + + mocker.patch("selenium.webdriver.ChromeOptions", return_value=mock_options) + + # Call the function + options = Firefox.creaet_options() + + # Ensure options were set + assert mock_options.add_argument.called + assert options == mock_options \ No newline at end of file From 03f387f32256d2fa32d407f892c29a45e09e0665 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Tue, 12 Nov 2024 00:46:33 -0800 Subject: [PATCH 09/16] test: update test cases --- tests/test_utils.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 2aa2986c7..8f375dde1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,8 @@ from unittest import mock from selenium.webdriver.remote.webelement import WebElement from src.utils.browser_utils import is_scrollable, scroll_slow +from src.webdrivers.base_browser import BrowserType, BrowserProfile +from src.webdrivers.browser_factory import BrowserFactory from src.webdrivers.chrome import Chrome from src.webdrivers.firefox import Firefox @@ -14,15 +16,17 @@ def mock_logger(mocker): mocker.patch("src.logging.logger") # Test ensure_chrome_profile function -def test_ensure_chrome_profile(mocker): +def test_ensure_browser_profiles(mocker): mocker.patch("os.path.exists", return_value=False) # Pretend directory doesn't exist mocker.patch("os.makedirs") # Mock making directories # Call the function - profile_path = ensure_chrome_profile() + chrome_profile_path = BrowserProfile(BrowserType.CHROME).ensure_profile_exists() + firefox_profile_path = BrowserProfile(BrowserType.FIREFOX).ensure_profile_exists() # Verify that os.makedirs was called twice to create the directory - assert profile_path.endswith("linkedin_profile") + assert chrome_profile_path.endswith("linkedin_profile") + assert firefox_profile_path.endswith("linkedin_profile") assert os.path.exists.called assert os.makedirs.called @@ -71,7 +75,6 @@ def test_scroll_slow_element_not_scrollable(mocker): # Test chrome_browser_options function def test_chrome_browser_options(mocker): - mocker.patch("src.utils.chrome_utils.ensure_chrome_profile") mocker.patch("os.path.dirname", return_value="/mocked/path") mocker.patch("os.path.basename", return_value="profile_directory") @@ -80,7 +83,7 @@ def test_chrome_browser_options(mocker): mocker.patch("selenium.webdriver.ChromeOptions", return_value=mock_options) # Call the function - options = Chrome.creaet_options() + options = Chrome().create_options() # Ensure options were set assert mock_options.add_argument.called @@ -88,17 +91,16 @@ def test_chrome_browser_options(mocker): # Test firefox_browser_options function def test_firefox_browser_options(mocker): - mocker.patch("src.utils.chrome_utils.ensure_chrome_profile") mocker.patch("os.path.dirname", return_value="/mocked/path") mocker.patch("os.path.basename", return_value="profile_directory") mock_options = mocker.Mock() - mocker.patch("selenium.webdriver.ChromeOptions", return_value=mock_options) + mocker.patch("selenium.webdriver.FirefoxOptions", return_value=mock_options) # Call the function - options = Firefox.creaet_options() + options = Firefox().create_options() # Ensure options were set assert mock_options.add_argument.called - assert options == mock_options \ No newline at end of file + assert options == mock_options From af7271ab5f03404d5aa90f8325c38b26404a66a5 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:46:12 -0800 Subject: [PATCH 10/16] refactor: simplify the browser factory --- main.py | 1 - src/webdrivers/base_browser.py | 4 ++-- src/webdrivers/browser_factory.py | 18 +++--------------- tests/test_utils.py | 7 +++---- 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/main.py b/main.py index 112c9a8ce..adca46d8b 100644 --- a/main.py +++ b/main.py @@ -212,7 +212,6 @@ def main(collect: bool = False, resume: Optional[Path] = None): parameters['outputFileDirectory'] = output_folder parameters['collectMode'] = collect # check if the config is not set as well - BrowserFactory.init_browser_type() create_and_run_bot(parameters, llm_api_key) except ConfigError as ce: diff --git a/src/webdrivers/base_browser.py b/src/webdrivers/base_browser.py index 32dc9beaa..7757b8fbb 100644 --- a/src/webdrivers/base_browser.py +++ b/src/webdrivers/base_browser.py @@ -7,8 +7,8 @@ class BrowserType(Enum): """Enum for supported browser types""" - CHROME = 'chrome' - FIREFOX = 'firefox' + CHROME = "chrome" + FIREFOX = "firefox" class BrowserProfile: """Manages browser profile creation and configuration""" diff --git a/src/webdrivers/browser_factory.py b/src/webdrivers/browser_factory.py index 29ee78ae5..f3e9aa0a2 100644 --- a/src/webdrivers/browser_factory.py +++ b/src/webdrivers/browser_factory.py @@ -5,18 +5,11 @@ from config import BROWSER_TYPE_CONFIG from src.webdrivers.base_browser import BrowserType -from src.webdrivers.chrome import Chrome -from src.webdrivers.firefox import Firefox class BrowserFactory: """Factory class for creating browser instances""" - _browser_type: BrowserType = BrowserType.CHROME # Default browser type - - _browsers = { - BrowserType.CHROME: Chrome, - BrowserType.FIREFOX: Firefox - } + _browser_type: BrowserType = BROWSER_TYPE_CONFIG @classmethod def get_browser_type(cls) -> BrowserType: @@ -26,16 +19,11 @@ def get_browser_type(cls) -> BrowserType: @classmethod def set_browser_type(cls, browser_type: BrowserType) -> None: """Set browser type""" - if browser_type not in cls._browsers: + if browser_type not in BrowserType.__members__: raise ValueError(f"Unsupported browser type: {browser_type}") cls._browser_type = browser_type logger.debug(f"Browser type set to: {browser_type}") - @classmethod - def init_browser_type(cls) -> None: - """Initialize browser type from the config""" - cls.set_browser_type(BROWSER_TYPE_CONFIG) - @classmethod def get_driver(cls, browser_type: BrowserType) -> Union[webdriver.Chrome, webdriver.Firefox]: """ @@ -47,7 +35,7 @@ def get_driver(cls, browser_type: BrowserType) -> Union[webdriver.Chrome, webdri Raises: RuntimeError: If browser initialization fails """ - browser_class = cls._browsers.get(browser_type) + browser_class = browser_type.value if not browser_class: raise ValueError(f"Unsupported browser type: {browser_type}") diff --git a/tests/test_utils.py b/tests/test_utils.py index 8f375dde1..6b3e51263 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,7 +6,6 @@ from selenium.webdriver.remote.webelement import WebElement from src.utils.browser_utils import is_scrollable, scroll_slow from src.webdrivers.base_browser import BrowserType, BrowserProfile -from src.webdrivers.browser_factory import BrowserFactory from src.webdrivers.chrome import Chrome from src.webdrivers.firefox import Firefox @@ -91,10 +90,9 @@ def test_chrome_browser_options(mocker): # Test firefox_browser_options function def test_firefox_browser_options(mocker): - mocker.patch("os.path.dirname", return_value="/mocked/path") - mocker.patch("os.path.basename", return_value="profile_directory") - mock_options = mocker.Mock() + mock_profile = mocker.Mock(spec=BrowserProfile) + mock_profile.profile_path = "/mocked/path" mocker.patch("selenium.webdriver.FirefoxOptions", return_value=mock_options) @@ -103,4 +101,5 @@ def test_firefox_browser_options(mocker): # Ensure options were set assert mock_options.add_argument.called + assert mock_options.set_preference.called assert options == mock_options From 74ac86eaa4f371291a439952f52c55022a5c5232 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:46:28 -0800 Subject: [PATCH 11/16] refactor: remove unused package --- src/ai_hawk/job_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ai_hawk/job_manager.py b/src/ai_hawk/job_manager.py index 59d3147dd..d090a4223 100644 --- a/src/ai_hawk/job_manager.py +++ b/src/ai_hawk/job_manager.py @@ -4,7 +4,6 @@ import time from itertools import product from pathlib import Path -from turtle import color from inputimeout import inputimeout, TimeoutOccurred from selenium.common.exceptions import NoSuchElementException From e524214e978a748b15eea48153834bc269cfbabd Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:03:11 -0800 Subject: [PATCH 12/16] refactor: browser factory --- src/webdrivers/browser_factory.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/webdrivers/browser_factory.py b/src/webdrivers/browser_factory.py index f3e9aa0a2..f39c4e1d3 100644 --- a/src/webdrivers/browser_factory.py +++ b/src/webdrivers/browser_factory.py @@ -5,6 +5,8 @@ from config import BROWSER_TYPE_CONFIG from src.webdrivers.base_browser import BrowserType +from src.webdrivers.chrome import Chrome +from src.webdrivers.firefox import Firefox class BrowserFactory: @@ -35,9 +37,11 @@ def get_driver(cls, browser_type: BrowserType) -> Union[webdriver.Chrome, webdri Raises: RuntimeError: If browser initialization fails """ - browser_class = browser_type.value - if not browser_class: + if browser_type == BrowserType.CHROME: + browser = Chrome() + elif browser_type == BrowserType.FIREFOX: + browser = Firefox() + else: raise ValueError(f"Unsupported browser type: {browser_type}") - browser = browser_class() return browser.create_driver() From 5de4d16bda44dc86d0ff079ee8e0b8e81bbee170 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Thu, 14 Nov 2024 02:35:09 -0800 Subject: [PATCH 13/16] refactor: remove duplicates and using singleton --- main.py | 11 +---------- src/webdrivers/browser_factory.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/main.py b/main.py index adca46d8b..c9d37515d 100644 --- a/main.py +++ b/main.py @@ -4,12 +4,10 @@ from pathlib import Path import yaml import click -from selenium import webdriver from selenium.common.exceptions import WebDriverException from lib_resume_builder_AIHawk import Resume, FacadeManager, ResumeGenerator, StyleManager from typing import Optional -from src.webdrivers.base_browser import BrowserType from src.webdrivers.browser_factory import BrowserFactory from src.job_application_profile import JobApplicationProfile from src.logging import logger @@ -155,12 +153,6 @@ def file_paths_to_dict(resume_file: Path | None, plain_text_resume_file: Path) - return result -def init_browser(browser_type: BrowserType = BrowserType.CHROME) -> webdriver.Chrome | webdriver.Firefox: - try: - return BrowserFactory.get_driver(browser_type) - except Exception as e: - raise RuntimeError(f"Failed to initialize browser: {str(e)}") - def create_and_run_bot(parameters, llm_api_key): try: style_manager = StyleManager() @@ -176,7 +168,7 @@ def create_and_run_bot(parameters, llm_api_key): job_application_profile_object = JobApplicationProfile(plain_text_resume) - browser = init_browser(BrowserFactory.get_browser_type()) + browser = BrowserFactory.get_browser() login_component = get_authenticator(driver=browser, platform='linkedin') apply_component = AIHawkJobManager(browser) gpt_answerer_component = GPTAnswerer(parameters, llm_api_key) @@ -211,7 +203,6 @@ def main(collect: bool = False, resume: Optional[Path] = None): parameters['uploads'] = FileManager.file_paths_to_dict(resume, plain_text_resume_file) parameters['outputFileDirectory'] = output_folder parameters['collectMode'] = collect - # check if the config is not set as well create_and_run_bot(parameters, llm_api_key) except ConfigError as ce: diff --git a/src/webdrivers/browser_factory.py b/src/webdrivers/browser_factory.py index f39c4e1d3..45b412eeb 100644 --- a/src/webdrivers/browser_factory.py +++ b/src/webdrivers/browser_factory.py @@ -8,10 +8,13 @@ from src.webdrivers.chrome import Chrome from src.webdrivers.firefox import Firefox - class BrowserFactory: """Factory class for creating browser instances""" _browser_type: BrowserType = BROWSER_TYPE_CONFIG + _browsers = { + BrowserType.CHROME: Chrome, + BrowserType.FIREFOX: Firefox, + } @classmethod def get_browser_type(cls) -> BrowserType: @@ -27,7 +30,7 @@ def set_browser_type(cls, browser_type: BrowserType) -> None: logger.debug(f"Browser type set to: {browser_type}") @classmethod - def get_driver(cls, browser_type: BrowserType) -> Union[webdriver.Chrome, webdriver.Firefox]: + def get_browser(cls) -> Union[webdriver.Chrome, webdriver.Firefox]: """ Create and return a WebDriver instance for the specified browser type Args: @@ -37,11 +40,10 @@ def get_driver(cls, browser_type: BrowserType) -> Union[webdriver.Chrome, webdri Raises: RuntimeError: If browser initialization fails """ - if browser_type == BrowserType.CHROME: - browser = Chrome() - elif browser_type == BrowserType.FIREFOX: - browser = Firefox() - else: - raise ValueError(f"Unsupported browser type: {browser_type}") + browser_type = cls._browsers.get(cls._browser_type) + if browser_type is None: + raise ValueError(f"Unsupported browser type: {cls._browser_type}") + + browser = browser_type() return browser.create_driver() From dc875e67b0a1f74377cd3e4a463a32c8c5739719 Mon Sep 17 00:00:00 2001 From: Akhil Date: Fri, 15 Nov 2024 10:41:51 -0500 Subject: [PATCH 14/16] Update browser_factory.py --- src/webdrivers/browser_factory.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/webdrivers/browser_factory.py b/src/webdrivers/browser_factory.py index 45b412eeb..1477ca76b 100644 --- a/src/webdrivers/browser_factory.py +++ b/src/webdrivers/browser_factory.py @@ -11,10 +11,6 @@ class BrowserFactory: """Factory class for creating browser instances""" _browser_type: BrowserType = BROWSER_TYPE_CONFIG - _browsers = { - BrowserType.CHROME: Chrome, - BrowserType.FIREFOX: Firefox, - } @classmethod def get_browser_type(cls) -> BrowserType: @@ -23,8 +19,8 @@ def get_browser_type(cls) -> BrowserType: @classmethod def set_browser_type(cls, browser_type: BrowserType) -> None: - """Set browser type""" - if browser_type not in BrowserType.__members__: + # safety check additional to type check. + if browser_type not in BrowserType: raise ValueError(f"Unsupported browser type: {browser_type}") cls._browser_type = browser_type logger.debug(f"Browser type set to: {browser_type}") @@ -40,10 +36,7 @@ def get_browser(cls) -> Union[webdriver.Chrome, webdriver.Firefox]: Raises: RuntimeError: If browser initialization fails """ - browser_type = cls._browsers.get(cls._browser_type) - if browser_type is None: - raise ValueError(f"Unsupported browser type: {cls._browser_type}") - + browser = browser_type() return browser.create_driver() From bca071f1035775ce59f63901f3cc2690dab4cb76 Mon Sep 17 00:00:00 2001 From: Tverous <22272655+Tverous@users.noreply.github.com> Date: Fri, 15 Nov 2024 23:11:45 -0800 Subject: [PATCH 15/16] refactor: browser type --- config.py | 2 +- src/webdrivers/base_browser.py | 10 ++-------- src/webdrivers/browser_factory.py | 13 +++++++------ src/webdrivers/browser_type.py | 10 ++++++++++ src/webdrivers/chrome.py | 6 +++--- src/webdrivers/firefox.py | 7 ++----- tests/test_utils.py | 7 ++++--- 7 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 src/webdrivers/browser_type.py diff --git a/config.py b/config.py index 8d3d31e96..155b76c18 100644 --- a/config.py +++ b/config.py @@ -1,7 +1,7 @@ # In this file, you can set the configurations of the app. from constants import DEBUG, LLM_MODEL, OPENAI -from src.webdrivers.base_browser import BrowserType +from src.webdrivers.browser_type import BrowserType #config related to logging must have prefix LOG_ LOG_LEVEL = DEBUG diff --git a/src/webdrivers/base_browser.py b/src/webdrivers/base_browser.py index 7757b8fbb..fd9589bc9 100644 --- a/src/webdrivers/base_browser.py +++ b/src/webdrivers/base_browser.py @@ -1,19 +1,13 @@ import os from abc import ABC, abstractmethod -from enum import Enum from loguru import logger -class BrowserType(Enum): - """Enum for supported browser types""" - CHROME = "chrome" - FIREFOX = "firefox" - class BrowserProfile: """Manages browser profile creation and configuration""" - def __init__(self, browser_type: BrowserType): - self.browser_type: BrowserType = browser_type.name.lower() + def __init__(self, browser_type: str): + self.browser_type: str = browser_type.lower() self.profile_path = os.path.join( os.getcwd(), f"{self.browser_type}_profile", diff --git a/src/webdrivers/browser_factory.py b/src/webdrivers/browser_factory.py index 1477ca76b..2e1c8e801 100644 --- a/src/webdrivers/browser_factory.py +++ b/src/webdrivers/browser_factory.py @@ -4,14 +4,12 @@ from loguru import logger from config import BROWSER_TYPE_CONFIG -from src.webdrivers.base_browser import BrowserType -from src.webdrivers.chrome import Chrome -from src.webdrivers.firefox import Firefox +from src.webdrivers.browser_type import BrowserType + class BrowserFactory: """Factory class for creating browser instances""" _browser_type: BrowserType = BROWSER_TYPE_CONFIG - @classmethod def get_browser_type(cls) -> BrowserType: """Get current browser type""" @@ -19,6 +17,7 @@ def get_browser_type(cls) -> BrowserType: @classmethod def set_browser_type(cls, browser_type: BrowserType) -> None: + """Set browser type""" # safety check additional to type check. if browser_type not in BrowserType: raise ValueError(f"Unsupported browser type: {browser_type}") @@ -36,7 +35,9 @@ def get_browser(cls) -> Union[webdriver.Chrome, webdriver.Firefox]: Raises: RuntimeError: If browser initialization fails """ - - browser = browser_type() + if cls._browser_type not in BrowserType: + raise ValueError("Unsupported browser type: {cls._browser_type}") + + browser = cls._browser_type.value() return browser.create_driver() diff --git a/src/webdrivers/browser_type.py b/src/webdrivers/browser_type.py new file mode 100644 index 000000000..20b85b87b --- /dev/null +++ b/src/webdrivers/browser_type.py @@ -0,0 +1,10 @@ +from enum import Enum + +from src.webdrivers.chrome import Chrome +from src.webdrivers.firefox import Firefox + + +class BrowserType(Enum): + """Enum for supported browser types""" + CHROME = Chrome + FIREFOX = Firefox diff --git a/src/webdrivers/chrome.py b/src/webdrivers/chrome.py index 5b1b17d1f..29f72b374 100644 --- a/src/webdrivers/chrome.py +++ b/src/webdrivers/chrome.py @@ -1,16 +1,16 @@ import os + from loguru import logger from selenium import webdriver from selenium.webdriver.chrome.service import Service as ChromeService from webdriver_manager.chrome import ChromeDriverManager -from src.webdrivers.base_browser import Browser, BrowserType +from src.webdrivers.base_browser import Browser -chromeProfilePath = os.path.join(os.getcwd(), "chrome_profile", "linkedin_profile") class Chrome(Browser): """Chrome browser implementation""" - browser_type: BrowserType = BrowserType.CHROME + browser_type: str = "chrome" def create_options(self) -> webdriver.ChromeOptions: """Create Chrome-specific options""" diff --git a/src/webdrivers/firefox.py b/src/webdrivers/firefox.py index 9660e049d..09feefb3f 100644 --- a/src/webdrivers/firefox.py +++ b/src/webdrivers/firefox.py @@ -1,17 +1,14 @@ -import os - from loguru import logger from selenium import webdriver from selenium.webdriver.firefox.service import Service as FirefoxService from webdriver_manager.firefox import GeckoDriverManager -from src.webdrivers.base_browser import Browser, BrowserType +from src.webdrivers.base_browser import Browser -firefoxProfilePath = os.path.join(os.getcwd(), "firefox_profile", "linkedin_profile") class Firefox(Browser): """Firefox browser implementation""" - browser_type: BrowserType = BrowserType.FIREFOX + browser_type: str = "firefox" def create_options(self) -> webdriver.FirefoxOptions: """Create Firefox-specific options""" diff --git a/tests/test_utils.py b/tests/test_utils.py index 6b3e51263..f02af1374 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,8 @@ from unittest import mock from selenium.webdriver.remote.webelement import WebElement from src.utils.browser_utils import is_scrollable, scroll_slow -from src.webdrivers.base_browser import BrowserType, BrowserProfile +from src.webdrivers.base_browser import BrowserProfile +from src.webdrivers.browser_type import BrowserType from src.webdrivers.chrome import Chrome from src.webdrivers.firefox import Firefox @@ -20,8 +21,8 @@ def test_ensure_browser_profiles(mocker): mocker.patch("os.makedirs") # Mock making directories # Call the function - chrome_profile_path = BrowserProfile(BrowserType.CHROME).ensure_profile_exists() - firefox_profile_path = BrowserProfile(BrowserType.FIREFOX).ensure_profile_exists() + chrome_profile_path = BrowserProfile(BrowserType.CHROME.name).ensure_profile_exists() + firefox_profile_path = BrowserProfile(BrowserType.FIREFOX.name).ensure_profile_exists() # Verify that os.makedirs was called twice to create the directory assert chrome_profile_path.endswith("linkedin_profile") From 19b36c1a8a399b33ac3ca9db894acd052f7d4e43 Mon Sep 17 00:00:00 2001 From: Akhil Date: Mon, 2 Dec 2024 17:44:10 -0500 Subject: [PATCH 16/16] test code changes, fixed deprication issue --- src/webdrivers/firefox.py | 3 +-- tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/webdrivers/firefox.py b/src/webdrivers/firefox.py index 09feefb3f..6e3cb60e7 100644 --- a/src/webdrivers/firefox.py +++ b/src/webdrivers/firefox.py @@ -36,8 +36,7 @@ def create_options(self) -> webdriver.FirefoxOptions: options.set_preference(key, value) if self.profile.profile_path: - profile = webdriver.FirefoxProfile(self.profile.profile_path) - options.profile = profile + options.set_preference("profile", self.profile.profile_path) logger.debug(f"Using Firefox profile directory: {self.profile.profile_path}") else: options.set_preference("browser.privatebrowsing.autostart", True) diff --git a/tests/test_utils.py b/tests/test_utils.py index f02af1374..1cb09e6c6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -75,7 +75,7 @@ def test_scroll_slow_element_not_scrollable(mocker): # Test chrome_browser_options function def test_chrome_browser_options(mocker): - mocker.patch("os.path.dirname", return_value="/mocked/path") + mocker.patch("os.path.dirname", return_value="mocked/path") mocker.patch("os.path.basename", return_value="profile_directory") mock_options = mocker.Mock()