chore: release v0.0.5

docker-compose.yml

@@ -64,8 +64,6 @@ services:
       - redis
       - minio
     volumes:
-      - ./server:/app/server      # Mount server source code for hot reloading
-      - ./maxun-core:/app/maxun-core  # Mount maxun-core for any shared code updates
       - /var/run/dbus:/var/run/dbus
 
   frontend:
@@ -79,13 +77,10 @@ services:
     environment:
       PUBLIC_URL: ${PUBLIC_URL}
       BACKEND_URL: ${BACKEND_URL}
-    volumes:
-      - ./:/app             # Mount entire frontend app directory for hot reloading
-      - /app/node_modules   # Anonymous volume to prevent overwriting node_modules
     depends_on:
       - backend
 
 volumes:
   postgres_data:
   minio_data:
-  redis_data:
+  redis_data:

maxun-core/src/browserSide/scraper.js

@@ -265,41 +265,72 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
     const scrapedData = [];
 
     while (scrapedData.length < limit) {
-      // Get all parent elements matching the listSelector
-      const parentElements = Array.from(document.querySelectorAll(listSelector));
-
-      // Iterate through each parent element
-      for (const parent of parentElements) {
-        if (scrapedData.length >= limit) break;
-        const record = {};
-
-        // For each field, select the corresponding element within the parent
-        for (const [label, { selector, attribute }] of Object.entries(fields)) {
-          const fieldElement = parent.querySelector(selector);
-
-          if (fieldElement) {
-            if (attribute === 'innerText') {
-              record[label] = fieldElement.innerText.trim();
-            } else if (attribute === 'innerHTML') {
-              record[label] = fieldElement.innerHTML.trim();
-            } else if (attribute === 'src') {
-              // Handle relative 'src' URLs
-              const src = fieldElement.getAttribute('src');
-              record[label] = src ? new URL(src, window.location.origin).href : null;
-            } else if (attribute === 'href') {
-              // Handle relative 'href' URLs
-              const href = fieldElement.getAttribute('href');
-              record[label] = href ? new URL(href, window.location.origin).href : null;
-            } else {
-              record[label] = fieldElement.getAttribute(attribute);
+        let parentElements = Array.from(document.querySelectorAll(listSelector));
+
+        // If we only got one element or none, try a more generic approach
+        if (limit > 1 && parentElements.length <= 1) {
+            const [containerSelector, _] = listSelector.split('>').map(s => s.trim());
+            const container = document.querySelector(containerSelector);
+
+            if (container) {
+                const allChildren = Array.from(container.children);
+
+                const firstMatch = document.querySelector(listSelector);
+                if (firstMatch) {
+                    // Get classes from the first matching element
+                    const firstMatchClasses = Array.from(firstMatch.classList);
+
+                    // Find similar elements by matching most of their classes
+                    parentElements = allChildren.filter(element => {
+                        const elementClasses = Array.from(element.classList);
+
+                        // Element should share at least 70% of classes with the first match
+                        const commonClasses = firstMatchClasses.filter(cls => 
+                            elementClasses.includes(cls));
+                        return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7);
+                    });
+                }
             }
-          }
         }
-        scrapedData.push(record);
-      }
+
+        // Iterate through each parent element
+        for (const parent of parentElements) {
+            if (scrapedData.length >= limit) break;
+            const record = {};
+
+            // For each field, select the corresponding element within the parent
+            for (const [label, { selector, attribute }] of Object.entries(fields)) {
+                const fieldElement = parent.querySelector(selector);
+
+                if (fieldElement) {
+                    if (attribute === 'innerText') {
+                        record[label] = fieldElement.innerText.trim();
+                    } else if (attribute === 'innerHTML') {
+                        record[label] = fieldElement.innerHTML.trim();
+                    } else if (attribute === 'src') {
+                        // Handle relative 'src' URLs
+                        const src = fieldElement.getAttribute('src');
+                        record[label] = src ? new URL(src, window.location.origin).href : null;
+                    } else if (attribute === 'href') {
+                        // Handle relative 'href' URLs
+                        const href = fieldElement.getAttribute('href');
+                        record[label] = href ? new URL(href, window.location.origin).href : null;
+                    } else {
+                        record[label] = fieldElement.getAttribute(attribute);
+                    }
+                }
+            }
+            scrapedData.push(record);
+        }
+
+        // If we've processed all available elements and still haven't reached the limit,
+        // break to avoid infinite loop
+        if (parentElements.length === 0 || scrapedData.length >= parentElements.length) {
+            break;
+        }
     }
-    return scrapedData
-  };
+    return scrapedData;
+};
 
 
   /**

maxun-core/src/interpret.ts

@@ -192,8 +192,8 @@ export default class Interpreter extends EventEmitter {
     // const actionable = async (selector: string): Promise<boolean> => {
     //   try {
     //     const proms = [
-    //       page.isEnabled(selector, { timeout: 5000 }),
-    //       page.isVisible(selector, { timeout: 5000 }),
+    //       page.isEnabled(selector, { timeout: 10000 }),
+    //       page.isVisible(selector, { timeout: 10000 }),
     //     ];
 
     //     return await Promise.all(proms).then((bools) => bools.every((x) => x));
@@ -214,6 +214,17 @@ export default class Interpreter extends EventEmitter {
     //     return [];
     //   }),
     // ).then((x) => x.flat());
+
+    const presentSelectors: SelectorArray = await Promise.all(
+        selectors.map(async (selector) => {
+            try {
+                await page.waitForSelector(selector, { state: 'attached' });
+                return [selector];
+            } catch (e) {
+                return [];
+            }
+        }),
+    ).then((x) => x.flat());
 
     const action = workflowCopy[workflowCopy.length - 1];
 
@@ -233,7 +244,7 @@ export default class Interpreter extends EventEmitter {
             ...p,
             [cookie.name]: cookie.value,
           }), {}),
-      selectors,
+      selectors: presentSelectors,
     };
   }
 
@@ -767,6 +778,8 @@ export default class Interpreter extends EventEmitter {
   public async run(page: Page, params?: ParamType): Promise<void> {
     this.log('Starting the workflow.', Level.LOG);
     const context = page.context();
+
+    page.setDefaultNavigationTimeout(100000);
 
     // Check proxy settings from context options
     const contextOptions = (context as any)._options;

server/src/browser-management/classes/RemoteBrowser.ts

@@ -104,7 +104,7 @@ export class RemoteBrowser {
         } catch {
             return url;
         }
-    }   
+    }
 
     /**
      * Determines if a URL change is significant enough to emit
@@ -130,11 +130,11 @@ export class RemoteBrowser {
         });
 
         // Handle page load events with retry mechanism
-        page.on('load', async () => { 
+        page.on('load', async () => {
             const injectScript = async (): Promise<boolean> => {
                 try {
                     await page.waitForLoadState('networkidle', { timeout: 5000 });
-                    
+
                     await page.evaluate(getInjectableScript());
                     return true;
                 } catch (error: any) {
@@ -148,44 +148,37 @@ export class RemoteBrowser {
         });
     }
 
+    private getUserAgent() {
+        const userAgents = [
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:117.0) Gecko/20100101 Firefox/117.0',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.1938.81 Safari/537.36 Edg/116.0.1938.81',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.96 Safari/537.36 OPR/101.0.4843.25',
+            'Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.62 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:118.0) Gecko/20100101 Firefox/118.0',
+        ];
+
+        return userAgents[Math.floor(Math.random() * userAgents.length)];
+    }
+
     /**
      * An asynchronous constructor for asynchronously initialized properties.
      * Must be called right after creating an instance of RemoteBrowser class.
      * @param options remote browser options to be used when launching the browser
      * @returns {Promise<void>}
      */
     public initialize = async (userId: string): Promise<void> => {
-        // const launchOptions = {
-        //     headless: true,
-        //     proxy: options.launchOptions?.proxy,
-        //     chromiumSandbox: false,
-        //     args: [
-        //         '--no-sandbox',
-        //         '--disable-setuid-sandbox',
-        //         '--headless=new',
-        //         '--disable-gpu',
-        //         '--disable-dev-shm-usage',
-        //         '--disable-software-rasterizer',
-        //         '--in-process-gpu',
-        //         '--disable-infobars',
-        //         '--single-process', 
-        //         '--no-zygote',
-        //         '--disable-notifications',
-        //         '--disable-extensions',
-        //         '--disable-background-timer-throttling',
-        //         ...(options.launchOptions?.args || [])
-        //     ],
-        //     env: {
-        //         ...process.env,
-        //         CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new'
-        //     }
-        // };
-        // console.log('Launch options before:', options.launchOptions);
-        // this.browser = <Browser>(await options.browser.launch(launchOptions));
-
-        // console.log('Launch options after:', options.launchOptions)
         this.browser = <Browser>(await chromium.launch({
             headless: true,
+            args: [
+                "--disable-blink-features=AutomationControlled",
+                "--disable-web-security",
+                "--disable-features=IsolateOrigins,site-per-process",
+                "--disable-site-isolation-trials",
+                "--disable-extensions",
+                "--no-sandbox",
+                "--disable-dev-shm-usage",
+            ], 
         }));
         const proxyConfig = await getDecryptedProxyConfig(userId);
         let proxyOptions: { server: string, username?: string, password?: string } = { server: '' };
@@ -201,7 +194,7 @@ export class RemoteBrowser {
         const contextOptions: any = {
             viewport: { height: 400, width: 900 },
             // recordVideo: { dir: 'videos/' }
-             // Force reduced motion to prevent animation issues
+            // Force reduced motion to prevent animation issues
             reducedMotion: 'reduce',
             // Force JavaScript to be enabled
             javaScriptEnabled: true,
@@ -210,7 +203,8 @@ export class RemoteBrowser {
             // Disable hardware acceleration
             forcedColors: 'none',
             isMobile: false,
-            hasTouch: false
+            hasTouch: false,
+            userAgent: this.getUserAgent(),
         };
 
         if (proxyOptions.server) {
@@ -220,18 +214,37 @@ export class RemoteBrowser {
                 password: proxyOptions.password ? proxyOptions.password : undefined,
             };
         }
-        const browserUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.38 Safari/537.36";
-
 
-        contextOptions.userAgent = browserUserAgent;
         this.context = await this.browser.newContext(contextOptions);
+        await this.context.addInitScript(
+            `const defaultGetter = Object.getOwnPropertyDescriptor(
+              Navigator.prototype,
+              "webdriver"
+            ).get;
+            defaultGetter.apply(navigator);
+            defaultGetter.toString();
+            Object.defineProperty(Navigator.prototype, "webdriver", {
+              set: undefined,
+              enumerable: true,
+              configurable: true,
+              get: new Proxy(defaultGetter, {
+                apply: (target, thisArg, args) => {
+                  Reflect.apply(target, thisArg, args);
+                  return false;
+                },
+              }),
+            });
+            const patchedGetter = Object.getOwnPropertyDescriptor(
+              Navigator.prototype,
+              "webdriver"
+            ).get;
+            patchedGetter.apply(navigator);
+            patchedGetter.toString();`
+        );
         this.currentPage = await this.context.newPage();
 
         await this.setupPageEventListeners(this.currentPage);
 
-        // await this.currentPage.setExtraHTTPHeaders({
-        //     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
-        // });
         const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']);
         await blocker.enableBlockingInPage(this.currentPage);
         this.client = await this.currentPage.context().newCDPSession(this.currentPage);
@@ -456,7 +469,7 @@ export class RemoteBrowser {
         this.currentPage = newPage;
         if (this.currentPage) {
             await this.setupPageEventListeners(this.currentPage);
-            
+
             this.client = await this.currentPage.context().newCDPSession(this.currentPage);
             await this.subscribeToScreencast();
         } else {