problems with uploading elastic data

[Sovivka]

Greetings, i have problems with uploading elastic data with my script.
The essence of the script is that once every 5 minutes it polls the t-pot on port 64297, the address of the request is spelled out as https://tpotaddress:64297/es
The data is uploaded to the remote machine where the script itself is located.
The problem is as follows, when contacting during the day there are no results and only once a day, at midnight, all the ip addresses that have accessed the traps over the past day are unloaded at a time.
Can you tell me what this might be related to and maybe something needs to be changed in the elastic configuration so that the script picks up addresses every 5 minutes?

[dmille6]

you want port 64298 to access elasticsearch for queries.. you might have to open it up in the docker compose file.

i've got a similar script written in python, I pull every 15 min, filter and parse the results then submit to various open source cyber communities like alienvault:

so in my main.py

from lib_pull_tpot_data import PullTPOTData
#create python Object
# host would be something like: http://123.123.123.123:64298
# indexname would be "logstash-*"
tpotObj = PullTPOTData(es_server_host=host, indexname=es_hp_data_index)

# pull data
# timedelta  is an integer in minutes
results = tpotObj.pull_tpot_data(time_delta_minutes=time_delta_minutes)

the class that actually does the work:
lib_pull_tpot_data.py
`
class PullTPOTData:
all_results=[]

def __init__(self, es_server_host, indexname, batch_size=1000, scroll_time='2m'):
    """Initialize the TPOT data puller."""
    self.ignore_list = ["Fatt", "P0f", "Suricata", "NGINX", "p0f", "fatt", "suricata", "nginx"]
    self.es_server_host = es_server_host
    self.indexname = indexname
    self.batch_size = batch_size
    self.scroll_time = scroll_time

    try:
        self.es = Elasticsearch(self.es_server_host)
        logging.info(f'Connected to Elasticsearch at {self.es_server_host}')
    except exceptions.ConnectionError as e:
        logging.error(f"Error connecting to Elasticsearch: {e}")
    except exceptions.ElasticsearchException as e:
        logging.error(f"General Elasticsearch error: {e}")

def pull_tpot_data(self, time_delta_minutes=5):
    """Pull TPOT data from Elasticsearch within the specified time delta in minutes."""
    all_results = []
    time_delta = f"now-{time_delta_minutes}m"
    logging.info(f'Pulling TPOT data from {self.indexname} since {time_delta}')

    query = {
        "bool": {
            "must": [
                {
                    "range": {
                        "@timestamp": {
                            "gte": time_delta,
                            "lte": "now",
                            "format": "strict_date_optional_time"
                        }
                    }
                }
            ],
            "must_not": [
                {
                    "terms": {
                        "type": self.ignore_list
                    }
                }
            ]
        }
    }

    try:
        # Initialize the scroll
        response = self.es.search(
            index=self.indexname,
            body={"query": query},
            scroll=self.scroll_time,
            size=self.batch_size
        )
        scroll_id = response['_scroll_id']
        hits = response['hits']['hits']

        # Loop through all the results
        while hits:
            all_results.extend(hits)
            response = self.es.scroll(scroll_id=scroll_id, scroll=self.scroll_time)
            scroll_id = response['_scroll_id']
            hits = response['hits']['hits']

        # Clean up the scroll when done
        self.es.clear_scroll(scroll_id=scroll_id)
        logging.info(f"Successfully pulled {len(all_results)} records from {self.indexname}")

    except exceptions.NotFoundError as e:
        logging.error(f"Index not found: {e}")
    except exceptions.ConnectionTimeout as e:
        logging.error(f"Connection timeout during query: {e}")
    except exceptions.ElasticsearchException as e:
        logging.error(f"Error during Elasticsearch query: {e}", exc_info=True)
    except KeyError as e:
        logging.error(f"KeyError: {e} - possibly missing expected fields in the response", exc_info=True)
    except Exception as e:
        logging.error(f"Unexpected error: {e}", exc_info=True)

    self.all_results = all_results.copy()
    return all_results

`
I hope this helps