From 3faea7997cd691a56ebe4fc03967e32a8ab0a4bb Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 16 Dec 2024 10:59:17 +0100 Subject: [PATCH] fix: [crawler] fix reload_crawlers_stats queues stats --- bin/lib/crawlers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index e56293ab..4534a702 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1101,8 +1101,14 @@ def reload_crawlers_stats(): tasks = r_crawler.smembers(f'crawler:queue:type:{domain_type}') for task_uuid in tasks: task = CrawlerTask(task_uuid) - if not task.is_in_queue() and task.get_status() is None and not task.get_capture(): - task.delete() + if not task.is_in_queue() and task.get_status() is None: + capture = task.get_capture() + if capture: + c = CrawlerCapture(capture) + if not c.exists(): + task.delete() + else: + task.delete() #### Blocklist ####