google_trends(): parse news urls into urls_queue

deedy5 · Feb 17, 2024 · c7c4c42 · c7c4c42
1 parent 8a79ba1
commit c7c4c42
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/fake_traffic/fake_traffic.py b/fake_traffic/fake_traffic.py
@@ -169,6 +169,13 @@ def google_trends(self):
         elements = self.page.query_selector_all("//div[@class='title']")
         trends = [x for e in elements for x in e.inner_text().split(" • ")]
         logging.info(f"google_trends() GOT {len(trends)} trends")
+
+        for e in elements:
+            e.click()
+            self.page.wait_for_selector("//div[@class='carousel-wrapper']")
+            related_urls_elements = self.page.query_selector_all("//div[@class='carousel-wrapper']//a")
+            related_urls = [link.get_attribute("href") for link in related_urls_elements]
+            self.urls_queue.extend(related_urls)
         return trends
 
     def parse_urls(self, page, base_url):