From dc0545dfd072facaa25db440f3f9fe85634d237d Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Mar 2024 13:54:56 +0100 Subject: [PATCH] chg: [RetroHunt] retro hunt on messages --- bin/lib/chats_viewer.py | 57 +++++++++++++++++++ bin/lib/objects/Messages.py | 4 +- bin/lib/objects/ail_objects.py | 11 +++- bin/trackers/Retro_Hunt.py | 3 + .../templates/hunter/add_retro_hunt_task.html | 4 ++ 5 files changed, 75 insertions(+), 4 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 0ec42772..7f14cb45 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -323,6 +323,63 @@ def get_username_meta_from_global_id(username_global_id): username = Usernames.Username(username_id, instance_uuid) return username.get_meta() + +# TODO Filter +## Instance type +## Chats IDS +## SubChats IDS +## Threads IDS +## Daterange +def get_messages_iterator(filters={}): + + for instance_uuid in get_chat_service_instances(): + + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + + # subchannels + for subchannel_gid in chat.get_subchannels(): + _, _, subchannel_id = subchannel_gid.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid) + messages, _ = subchannel._get_messages(nb=-1) + for mess in messages: + _, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + # threads + + # threads + for threads in chat.get_threads(): + thread = ChatThreads.ChatThread(threads['id'], instance_uuid) + _, _ = thread._get_messages(nb=-1) + for mess in messages: + message_id, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + + # messages + messages, _ = chat._get_messages(nb=-1) + for mess in messages: + _, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + # threads ??? + +def get_nb_messages_iterator(filters={}): + nb_messages = 0 + for instance_uuid in get_chat_service_instances(): + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + # subchannels + for subchannel_gid in chat.get_subchannels(): + _, _, subchannel_id = subchannel_gid.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid) + nb_messages += subchannel.get_nb_messages() + # threads + for threads in chat.get_threads(): + thread = ChatThreads.ChatThread(threads['id'], instance_uuid) + nb_messages += thread.get_nb_messages() + # messages + nb_messages += chat.get_nb_messages() + return nb_messages + #### API #### def api_get_chat_service_instance(chat_instance_uuid): diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 56281e69..09645cf3 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -85,7 +85,8 @@ def get_content(self, r_type='str'): # TODO ADD cache # TODO Compress content ?? if r_type == 'str': return content elif r_type == 'bytes': - return content.encode() + if content: + return content.encode() def get_date(self): timestamp = self.get_timestamp() @@ -339,7 +340,6 @@ def create(obj_id, content, translation=None, tags=[]): message.create(content, translation=translation, tags=tags) return message - # TODO Encode translation diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 33413943..63248a79 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -18,6 +18,8 @@ from lib import btc_ail from lib import Tag +from lib import chats_viewer + from lib.objects import Chats from lib.objects import ChatSubChannels from lib.objects import ChatThreads @@ -32,7 +34,7 @@ from lib.objects import HHHashs from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Images -from lib.objects.Messages import Message +from lib.objects import Messages from lib.objects import Pgps from lib.objects.Screenshots import Screenshot from lib.objects import Titles @@ -90,7 +92,7 @@ def get_object(obj_type, subtype, obj_id): elif obj_type == 'image': return Images.Image(obj_id) elif obj_type == 'message': - return Message(obj_id) + return Messages.Message(obj_id) elif obj_type == 'screenshot': return Screenshot(obj_id) elif obj_type == 'title': @@ -296,6 +298,9 @@ def obj_iterator(obj_type, filters): return get_all_items_objects(filters=filters) elif obj_type == 'pgp': return Pgps.get_all_pgps_objects(filters=filters) + elif obj_type == 'message': + return chats_viewer.get_messages_iterator(filters=filters) + def card_objs_iterators(filters): nb = 0 @@ -310,6 +315,8 @@ def card_obj_iterator(obj_type, filters): return get_nb_items_objects(filters=filters) elif obj_type == 'pgp': return Pgps.nb_all_pgps_objects(filters=filters) + elif obj_type == 'message': + return chats_viewer.get_nb_messages_iterator(filters=filters) def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME """ diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index f6854781..6830a663 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -88,6 +88,9 @@ def compute(self, task_uuid): for obj in ail_objects.obj_iterator(obj_type, filters): self.obj = obj content = obj.get_content(r_type='bytes') + if not content: + continue + rule.match(data=content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) diff --git a/var/www/templates/hunter/add_retro_hunt_task.html b/var/www/templates/hunter/add_retro_hunt_task.html index d8763f61..fada045e 100644 --- a/var/www/templates/hunter/add_retro_hunt_task.html +++ b/var/www/templates/hunter/add_retro_hunt_task.html @@ -78,6 +78,10 @@
Create a new Retro Hunt task
+
+ + +
{#
#} {# #} {# #}