fix: [crawler] fix reload_crawlers_stats queues stats

ail-project · Dec 16, 2024 · 3faea79 · 3faea79
1 parent 5b5ee50
commit 3faea79
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py
@@ -1101,8 +1101,14 @@ def reload_crawlers_stats():
         tasks = r_crawler.smembers(f'crawler:queue:type:{domain_type}')
         for task_uuid in tasks:
             task = CrawlerTask(task_uuid)
-            if not task.is_in_queue() and task.get_status() is None and not task.get_capture():
-                task.delete()
+            if not task.is_in_queue() and task.get_status() is None:
+                capture = task.get_capture()
+                if capture:
+                    c = CrawlerCapture(capture)
+                    if not c.exists():
+                        task.delete()
+                else:
+                    task.delete()
 
 #### Blocklist ####