From c7c4c421359d262ff9d3f9e1f841bb0aceb1d6a6 Mon Sep 17 00:00:00 2001
From: deedy5 <65482418+deedy5@users.noreply.github.com>
Date: Sat, 17 Feb 2024 23:13:39 +0300
Subject: [PATCH] google_trends(): parse news urls into urls_queue

---
 fake_traffic/fake_traffic.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/fake_traffic/fake_traffic.py b/fake_traffic/fake_traffic.py
index 8e71d1e..6ca2da4 100644
--- a/fake_traffic/fake_traffic.py
+++ b/fake_traffic/fake_traffic.py
@@ -169,6 +169,13 @@ def google_trends(self):
         elements = self.page.query_selector_all("//div[@class='title']")
         trends = [x for e in elements for x in e.inner_text().split(" • ")]
         logging.info(f"google_trends() GOT {len(trends)} trends")
+
+        for e in elements:
+            e.click()
+            self.page.wait_for_selector("//div[@class='carousel-wrapper']")
+            related_urls_elements = self.page.query_selector_all("//div[@class='carousel-wrapper']//a")
+            related_urls = [link.get_attribute("href") for link in related_urls_elements]
+            self.urls_queue.extend(related_urls)
         return trends
 
     def parse_urls(self, page, base_url):