diff --git a/.github/workflows/unitTests.yaml b/.github/workflows/unitTests.yaml index 1547adb..3dfc21e 100644 --- a/.github/workflows/unitTests.yaml +++ b/.github/workflows/unitTests.yaml @@ -16,4 +16,4 @@ jobs: pip install pytest pip install pytest-cov - name: Run tests - run: PYTHONPATH=. pytest --cov-report term-missing --cov=src tests/UnitTests/test_*.py + run: PYTHONPATH=. pytest --cov-report term-missing --cov=src tests/UnitTests/test_*.py \ No newline at end of file diff --git a/README.md b/README.md index f6fa503..222e316 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,30 @@ By default `1password` API type has built in pagination settings and sets the `r | onepassword_limit | 1Password limit for number of events to return in a single request (allowed range: 100 to 1000) | Optional | 100 | | pagination_off | True if builtin pagination should be off, False otherwise | Optional | `False` | + + +
+ + Dockerhub + + +For dockerhub audit logs, use type `dockerhub` with the below parameters. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|------------------------|---------------------------------------------------------------------------------------|-------------------|-------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| dockerhub_user | DockerHub username | Required | - | +| dockerhub_token | DockerHub personal access token or password | Required | - | +| url | The request URL | Required | - | +| next_url | URL for the next page of results (used for pagination) | Optional | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| days_back_fetch | Number of days to fetch back in the first request. Adds a filter on `from` parameter. | Optional | -1 | +| refresh_token_interval | Interval in minutes to refresh the JWT token | Optional | 30 (minute) | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | + +
@@ -285,6 +309,8 @@ docker stop -t 30 logzio-api-fetcher ``` ## Changelog: +- **0.3.0**: + - Add support for dockerhub audit logs - **0.2.2**: - Resolve Azure mail reports bug - **0.2.1**: diff --git a/src/apis/dockerhub/Dockerhub.py b/src/apis/dockerhub/Dockerhub.py new file mode 100644 index 0000000..6747ec2 --- /dev/null +++ b/src/apis/dockerhub/Dockerhub.py @@ -0,0 +1,74 @@ +import time +from datetime import datetime, timedelta, UTC +import json +import logging +import requests +from pydantic import Field +from src.apis.general.Api import ApiFetcher + +logger = logging.getLogger(__name__) + + +class DockerHub(ApiFetcher): + """ + :param dockerhub_user: The DockerHub username + :param dockerhub_token: The DockerHub personal access token or password + :param days_back_fetch: Number of days to fetch back in the first request, Optional (adds a filter on 'from') + :param page_size: Number of events to return in a single request (for pagination) + :param refresh_token_interval: Interval in minutes to refresh the JWT token + """ + dockerhub_user: str = Field(frozen=True) + dockerhub_token: str = Field(frozen=False) + days_back_fetch: int = Field(default=-1, frozen=True) + refresh_token_interval: int = Field(default=30) + _jwt_token: str = None + _token_expiry: datetime = None + + def __init__(self, **data): + res_data_path = "logs" + headers = { + "Content-Type": "application/json", + } + super().__init__(headers=headers, response_data_path=res_data_path, **data) + self._initialize_params() + + def _initialize_params(self): + try: + params = {"page_size": 100} + if self.days_back_fetch > 0: + from_date = (datetime.now(UTC) - timedelta(days=self.days_back_fetch)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + params["from"] = from_date + query_string = "&".join([f"{key}={value}" for key, value in params.items()]) + if "?" in self.url: + self.url += f"&{query_string}" + else: + self.url += f"?{query_string}" + except Exception as e: + logger.error( + f"Failed to update request params. Sending {self.name} request with default params. Error: {e}") + + def _get_jwt_token(self): + if self._jwt_token and datetime.now(UTC) < self._token_expiry: + return self._jwt_token + + url = "https://hub.docker.com/v2/users/login" + payload = { + "username": self.dockerhub_user, + "password": self.dockerhub_token + } + headers = {"Content-Type": "application/json"} + try: + response = requests.post(url, json=payload, headers=headers) + response.raise_for_status() + token_response = response.json() + self._jwt_token = token_response.get("token") + self._token_expiry = datetime.now(UTC) + timedelta(minutes=self.refresh_token_interval) + return self._jwt_token + except requests.exceptions.RequestException as e: + logger.error(f"Failed to get JWT token: {e}") + + def send_request(self): + session_token = self._get_jwt_token() + self.headers["Authorization"] = f"Bearer {session_token}" + response = super().send_request() + return response diff --git a/src/apis/dockerhub/README.md b/src/apis/dockerhub/README.md new file mode 100644 index 0000000..7031716 --- /dev/null +++ b/src/apis/dockerhub/README.md @@ -0,0 +1,42 @@ +# DockerHub API Configuration +The `dockerhub` API type is used to fetch audit logs from DockerHub. It supports pagination and allows filtering logs based on a date range. + +## Configuration +| Parameter Name | Description | Required/Optional | Default | +|------------------------|---------------------------------------------------------------------------------------|-------------------|-------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| dockerhub_user | DockerHub username | Required | - | +| dockerhub_token | DockerHub personal access token or password | Required | - | +| url | The request URL | Required | - | +| next_url | URL for the next page of results (used for pagination) | Optional | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| days_back_fetch | Number of days to fetch back in the first request. Adds a filter on `from` parameter. | Optional | -1 | +| refresh_token_interval | Interval in minutes to refresh the JWT token | Optional | 30 (minute) | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | + +## Example +You can customize the endpoints to collect data from by adding extra API configurations under `apis`. DockerHub API Docs can be found [here](https://docs.docker.com/docker-hub/api/latest/). + +Example configuration: + +```yaml +apis: + - name: Dockerhub audit logs + type: dockerhub + dockerhub_token: <> + dockerhub_user: <> + url: https://hub.docker.com/v2/auditlogs/<> + next_url: https://hub.docker.com/v2/auditlogs/logzio?from={res.logs.[0].timestamp} + method: GET + days_back_fetch: 7 + scrape_interval: 1 + refresh_token_interval: 20 + additional_fields: + type: dockerhub-audit + eventType: auditevents + +logzio: + url: https://<>:8071 + token: <> +``` \ No newline at end of file diff --git a/src/apis/dockerhub/__init__.py b/src/apis/dockerhub/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/apis/dockerhub/test_e2e_dockerhub.py b/src/apis/dockerhub/test_e2e_dockerhub.py new file mode 100644 index 0000000..333d1e6 --- /dev/null +++ b/src/apis/dockerhub/test_e2e_dockerhub.py @@ -0,0 +1,101 @@ +import glob +import json +import os +import threading +from os.path import abspath, dirname +import requests +import unittest +import yaml + +from src.main import main + +TEST_TYPE = "dockerhub-audit-test" + +def _search_data(query): + """ + Send given search query to logzio and returns the result. + :param query: + :return: + """ + url = "https://api.logz.io/v1/search" + headers = { + "X-API-TOKEN": os.environ["LOGZIO_API_TOKEN"], + "CONTENT-TYPE": "application/json", + "ACCEPT": "application/json" + } + body = { + "query": { + "query_string": { + "query": query + } + } + } + + r = requests.post(url=url, headers=headers, json=body) + if r: + data = json.loads(r.text) + hits = data.get("hits").get("hits") + return hits + return [] + + +def delete_temp_files(): + """ + delete the temp config that generated for the test + """ + curr_path = abspath(dirname(dirname(__file__))) + test_configs_path = f"{curr_path}/testdata/*_temp.yaml" + + for file in glob.glob(test_configs_path): + os.remove(file) + + +def _update_config_tokens(file_path): + """ + Updates the tokens in the given file. + """ + with open(file_path, "r") as conf: + content = yaml.safe_load(conf) + e = os.environ + if "DOCKERHUB_TOKEN" not in os.environ: + raise EnvironmentError("DOCKERHUB_TOKEN environment variable is missing") + content["apis"][0]["dockerhub_token"] = os.environ["DOCKERHUB_TOKEN"] + + if "DOCKERHUB_USER" not in os.environ: + raise EnvironmentError("DOCKERHUB_USER environment variable is missing") + content["apis"][0]["dockerhub_user"] = os.environ["DOCKERHUB_USER"] + + if "LOGZIO_SHIPPING_TOKEN" not in os.environ: + raise EnvironmentError("LOGZIO_SHIPPING_TOKEN environment variable is missing") + content["logzio"]["token"] = os.environ["LOGZIO_SHIPPING_TOKEN"] + + path, ext = file_path.rsplit(".", 1) + temp_test_path = f"{path}_temp.{ext}" + + with open(temp_test_path, "w") as file: + yaml.dump(content, file) + + return temp_test_path + + +class TestDockerhubE2E(unittest.TestCase): + """ + Test data arrived to logzio + """ + + def test_data_in_logz(self): + curr_path = abspath(dirname(__file__)) + config_path = f"{curr_path}/testdata/valid_dockerhub_config.yaml" + temp_config_path = _update_config_tokens(config_path) + thread = threading.Thread(target=main, kwargs={"conf_path": temp_config_path}) + thread.daemon = True + thread.start() + thread.join(timeout=60) + azure_logs_in_acc = _search_data(f"type:{TEST_TYPE}") + self.assertTrue(azure_logs_in_acc) + self.assertTrue(all([log.get("_source").get("eventType") == "auditevents" for log in azure_logs_in_acc])) + delete_temp_files() + + +if __name__ == '__main__': + unittest.main() diff --git a/src/apis/dockerhub/testdata/valid_dockerhub_config.yaml b/src/apis/dockerhub/testdata/valid_dockerhub_config.yaml new file mode 100644 index 0000000..496eea9 --- /dev/null +++ b/src/apis/dockerhub/testdata/valid_dockerhub_config.yaml @@ -0,0 +1,20 @@ +apis: + - name: Dockerhub audit logs + type: dockerhub + dockerhub_token: token + dockerhub_user: user + url: https://hub.docker.com/v2/auditlogs/logzio + next_url: https://hub.docker.com/v2/auditlogs/logzio?from={res.logs.[0].timestamp} + method: GET + days_back_fetch: 7 + scrape_interval: 10 + additional_fields: + type: dockerhub-audit-test + eventType: auditevents + +logzio: + url: https://listener.logz.io:8071 + token: token + + + diff --git a/src/apis/dockerhub/utils/logging_config.ini b/src/apis/dockerhub/utils/logging_config.ini new file mode 100644 index 0000000..5cb5e41 --- /dev/null +++ b/src/apis/dockerhub/utils/logging_config.ini @@ -0,0 +1,22 @@ +[loggers] +keys = root + +[handlers] +keys = stream_handler + +[formatters] +keys = formatter + +[logger_root] +level = INFO +handlers = stream_handler + +[handler_stream_handler] +class = StreamHandler +level = INFO +formatter = formatter +args = (sys.stderr,) + +[formatter_formatter] +class=src.utils.MaskInfoFormatter.MaskInfoFormatter +format = %(asctime)s [%(levelname)s]: %(message)s diff --git a/src/config/ConfigReader.py b/src/config/ConfigReader.py index 4696a6a..224c264 100644 --- a/src/config/ConfigReader.py +++ b/src/config/ConfigReader.py @@ -11,6 +11,7 @@ from src.apis.cloudflare.Cloudflare import Cloudflare from src.apis.onepassword.OnePassword import OnePassword from src.output.LogzioShipper import LogzioShipper +from src.apis.dockerhub.Dockerhub import DockerHub INPUT_API_FIELD = "apis" OUTPUT_LOGZIO_FIELD = "logzio" @@ -21,7 +22,8 @@ "azure_graph": "AzureGraph", "azure_mail_reports": "AzureMailReports", "cloudflare": "Cloudflare", - "1password": "OnePassword" + "1password": "OnePassword", + "dockerhub": "DockerHub" } logger = logging.getLogger(__name__) diff --git a/tests/UnitTests/responsesExamples/dockerhub_audit_logs_res_example.json b/tests/UnitTests/responsesExamples/dockerhub_audit_logs_res_example.json new file mode 100644 index 0000000..1d29d37 --- /dev/null +++ b/tests/UnitTests/responsesExamples/dockerhub_audit_logs_res_example.json @@ -0,0 +1,75 @@ +{ + "logs": [ + { + "event_id": "1234abcd-5678-efgh-9101-ijklmnopqrst", + "account": "mock_account", + "action": "repo.tag.push", + "name": "mock_account/mock-repo", + "actor": "user123 - access token \"mock-token\"", + "data": { + "access_token_id": "abcd1234-ef56-7890-abcd-1234567890ef", + "access_token_label": "mock-token", + "access_token_type": "TYPE_UNSPECIFIED", + "digest": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "tag": "latest" + }, + "timestamp": "2024-11-01T12:34:56.789Z", + "action_description": "pushed the tag latest with the digest sha256:1234567890abc to the repository mock_account/mock-repo" + }, + { + "event_id": "5678ijkl-9101-mnop-2345-qrstuvwx5678", + "account": "mock_account", + "action": "repo.create", + "name": "mock_account/mock-new-repo", + "actor": "user456", + "data": { + "categories": "utilities", + "privacy": "public" + }, + "timestamp": "2024-11-02T10:11:12.345Z", + "action_description": "created the public repository mock_account/mock-new-repo" + }, + { + "event_id": "9101qrst-2345-uvwx-6789-yzabcdef1234", + "account": "mock_account", + "action": "repo.tag.push", + "name": "mock_account/mock-repo", + "actor": "user123 - access token \"mock-token\"", + "data": { + "access_token_id": "efgh5678-ijkl-9012-mnop-3456789012qr", + "access_token_label": "mock-token", + "access_token_type": "TYPE_UNSPECIFIED", + "digest": "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + "tag": "v1.0.1" + }, + "timestamp": "2024-11-03T08:23:45.678Z", + "action_description": "pushed the tag v1.0.1 with the digest sha256:abcdef1234567 to the repository mock_account/mock-repo" + }, + { + "event_id": "abcd5678-efgh-9101-ijkl-3456mnopqrst", + "account": "mock_account", + "action": "repo.delete", + "name": "mock_account/old-repo", + "actor": "user789", + "data": {}, + "timestamp": "2024-11-04T16:22:33.444Z", + "action_description": "deleted the repository mock_account/old-repo" + }, + { + "event_id": "efgh9101-ijkl-2345-mnop-6789qrstuvwx", + "account": "mock_account", + "action": "repo.tag.push", + "name": "mock_account/mock-repo", + "actor": "user123 - access token \"mock-token\"", + "data": { + "access_token_id": "ijkl9012-mnop-3456-qrst-7890uvwxyzab", + "access_token_label": "mock-token", + "access_token_type": "TYPE_UNSPECIFIED", + "digest": "sha256:7890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234", + "tag": "stable" + }, + "timestamp": "2024-11-05T14:45:23.567Z", + "action_description": "pushed the tag stable with the digest sha256:7890abcdef123 to the repository mock_account/mock-repo" + } + ] +} diff --git a/tests/UnitTests/test_dockerhub_audit_logs_api.py b/tests/UnitTests/test_dockerhub_audit_logs_api.py new file mode 100644 index 0000000..b397278 --- /dev/null +++ b/tests/UnitTests/test_dockerhub_audit_logs_api.py @@ -0,0 +1,171 @@ +from datetime import datetime, UTC, timedelta +import json +from os.path import abspath, dirname +from pydantic import ValidationError +import responses +import unittest + +from src.apis.dockerhub.Dockerhub import DockerHub + +curr_path = abspath(dirname(__file__)) + +default_dockerhub_config = { + "dockerhub_user": "some-user", + "dockerhub_token": "some-token", + "url": "https://hub.docker.com/v2/auditlogs/test", +} + + +class TestDockerHubApi(unittest.TestCase): + """ + Test cases for DockerHub API + """ + + def test_invalid_setup(self): + invalid_configs = [ + { + "dockerhub_user": default_dockerhub_config.get("dockerhub_user"), + "dockerhub_token": default_dockerhub_config.get("dockerhub_token"), + "data_request": { + "url": default_dockerhub_config.get("url") + } + }, + { + "dockerhub_user": default_dockerhub_config.get("dockerhub_user"), + "url": default_dockerhub_config.get("url") + }, + { + "dockerhub_token": default_dockerhub_config.get("dockerhub_token"), + }, + ] + + for config in invalid_configs: + with self.assertRaises(ValidationError): + DockerHub(**config) + + def test_valid_setup(self): + dh = DockerHub(**default_dockerhub_config) + + self.assertIn(default_dockerhub_config.get("url"), dh.url) + + self.assertEqual(dh.headers["Content-Type"], "application/json") + + self.assertEqual(dh.response_data_path, "logs") + + self.assertIsNone(dh._jwt_token) + + def test_invalid_days_back_fetch(self): + dh = DockerHub(**default_dockerhub_config, + days_back_fetch=-1) + self.assertNotIn("from=", dh.url) + + def test_url_with_existing_query(self): + dh = DockerHub(dockerhub_user=default_dockerhub_config.get("dockerhub_user"), + dockerhub_token=default_dockerhub_config.get("dockerhub_token"), + url="https://hub.docker.com/v2/auditlogs/test?existing_param=value", + days_back_fetch=1) + self.assertIn("&from=", dh.url) + self.assertIn("?existing_param=value", dh.url) + + def test_url_without_existing_query(self): + dh = DockerHub(**default_dockerhub_config, + days_back_fetch=1) + self.assertIn("&from=", dh.url) + + def test_start_date_generator(self): + zero_days_back = DockerHub(**default_dockerhub_config, + days_back_fetch=0) + + day_back = DockerHub(**default_dockerhub_config, + days_back_fetch=1) + + five_days_back = DockerHub(**default_dockerhub_config, + days_back_fetch=5) + + # Make sure the current format and needed dates are generated + from_date_zero_days_back = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + from_date_day_back = (datetime.now(UTC) - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + from_date_five_days_back = (datetime.now(UTC) - timedelta(days=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + self.assertNotIn(f"from={from_date_zero_days_back[:19]}", zero_days_back.url) + self.assertIn(f"from={from_date_day_back[:19]}", day_back.url) + self.assertIn(f"from={from_date_five_days_back[:19]}", five_days_back.url) + + @responses.activate + def test_dockerhub_send_request(self): + # Mock response from DockerHub API + with open(f"{curr_path}/responsesExamples/dockerhub_audit_logs_res_example.json", "r") as data_res_example_file: + data_res_body = json.loads(data_res_example_file.read()) + + # Mock token response + token_res_body = { + "token": "mocked_jwt_token", + } + responses.add(responses.POST, + "https://hub.docker.com/v2/users/login", + json=token_res_body, + status=200) + + responses.add(responses.GET, + default_dockerhub_config.get("url"), + json=data_res_body, + status=200) + + dh = DockerHub(**default_dockerhub_config) + result = dh.send_request() + + self.assertEqual(result, data_res_body.get("logs")) + self.assertEqual(dh.url, + default_dockerhub_config.get("url") + "?page_size=100") + + @responses.activate + def test_jwt_token_success(self): + token_res_body = { + "token": "mocked_jwt_token", + } + responses.add(responses.POST, + "https://hub.docker.com/v2/users/login", + json=token_res_body, + status=200) + + dh = DockerHub(**default_dockerhub_config) + token = dh._get_jwt_token() + + self.assertEqual(token, "mocked_jwt_token") + self.assertEqual(dh._jwt_token, "mocked_jwt_token") + + @responses.activate + def test_jwt_token_failure(self): + responses.add(responses.POST, + "https://hub.docker.com/v2/users/login", + status=401) + + dh = DockerHub(**default_dockerhub_config) + token = dh._get_jwt_token() + + self.assertIsNone(token) + self.assertIsNone(dh._jwt_token) + + @responses.activate + def test_send_request_unauthorized(self): + token_res_body = { + "token": "mocked_jwt_token", + } + responses.add(responses.POST, + "https://hub.docker.com/v2/users/login", + json=token_res_body, + status=200) + + responses.add(responses.GET, + default_dockerhub_config.get("url"), + status=401) + + dh = DockerHub(**default_dockerhub_config) + result = dh.send_request() + + self.assertEqual(result, []) + self.assertEqual(dh.headers["Authorization"], "Bearer mocked_jwt_token") + + +if __name__ == '__main__': + unittest.main()