diff --git a/fake_traffic/fake_traffic.py b/fake_traffic/fake_traffic.py index 8e71d1e..6ca2da4 100644 --- a/fake_traffic/fake_traffic.py +++ b/fake_traffic/fake_traffic.py @@ -169,6 +169,13 @@ def google_trends(self): elements = self.page.query_selector_all("//div[@class='title']") trends = [x for e in elements for x in e.inner_text().split(" • ")] logging.info(f"google_trends() GOT {len(trends)} trends") + + for e in elements: + e.click() + self.page.wait_for_selector("//div[@class='carousel-wrapper']") + related_urls_elements = self.page.query_selector_all("//div[@class='carousel-wrapper']//a") + related_urls = [link.get_attribute("href") for link in related_urls_elements] + self.urls_queue.extend(related_urls) return trends def parse_urls(self, page, base_url):