Fix Single Url Scrape

mendableai · May 15, 2024 · 3f9034b · 3f9034b
1 parent 4c767cd
commit 3f9034b
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/src/providers/WebScraper/single_url.ts b/src/providers/WebScraper/single_url.ts
@@ -35,6 +35,8 @@ export async function scrapSingleUrl(urlToScrap: string, toMarkdown: boolean = t
 
   try {
     let content = await scrapWithScrapingBee(urlToScrap);
+
+
 
     if (!content) {
       const res = await attemptScrapWithRequests(urlToScrap);
@@ -72,8 +74,14 @@ export async function scrapSingleUrl(urlToScrap: string, toMarkdown: boolean = t
       }
     });
 
+    if (formattedText.length < 1) {
+      formattedText = markdownContent;
+    }
+
     const text = sanitizeText(formattedText.trim());
 
+
+
     if (metadata) {
       // console.log(markdownContent)
       // console.log("here", toMarkdown)