From 884f19dfd892b89f7869593a0bed5be6eee481f0 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Thu, 12 Sep 2024 17:14:57 +0300 Subject: [PATCH] feat: add systemd journal logging configuration --- README.rst | 9 ++++++++ karapace.config.json | 1 + karapace/config.py | 2 ++ karapace/karapace_all.py | 43 ++++++++++++++++++++++++++++------- mypy.ini | 3 +++ pyproject.toml | 1 + requirements/requirements.txt | 2 ++ 7 files changed, 53 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 88bb1b249..846ee2e10 100644 --- a/README.rst +++ b/README.rst @@ -475,6 +475,15 @@ Keys to take special care are the ones needed to configure Kafka and advertised_ - ``true`` - If enabled, kafka errors which can be retried or custom errors specififed for the service will not be raised, instead, a warning log is emitted. This will denoise issue tracking systems, i.e. sentry + * - ``log_handler`` + - ``stdout`` + - Select the log handler. Default is standard output. Alternative log handler is ``systemd``. + * - ``log_level`` + - ``DEBUG`` + - Logging level. Default level is debug. + * - ``log_format`` + - ``%(name)-20s\t%(threadName)s\t%(levelname)-8s\t%(message)s`` + - Log format Authentication and authorization of Karapace Schema Registry REST API diff --git a/karapace.config.json b/karapace.config.json index 55303ff4d..2a3184a75 100644 --- a/karapace.config.json +++ b/karapace.config.json @@ -9,6 +9,7 @@ "group_id": "schema-registry", "host": "127.0.0.1", "log_level": "DEBUG", + "log_handler_class": "stdout", "port": 8081, "server_tls_certfile": null, "server_tls_keyfile": null, diff --git a/karapace/config.py b/karapace/config.py index bbae62701..c4609ce8a 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -48,6 +48,7 @@ class Config(TypedDict): registry_authfile: str | None rest_authorization: bool rest_base_uri: str | None + log_handler_class: str | None log_level: str log_format: str master_eligibility: bool @@ -124,6 +125,7 @@ class ConfigDefaults(Config, total=False): "registry_authfile": None, "rest_authorization": False, "rest_base_uri": None, + "log_handler": "stdout", "log_level": "DEBUG", "log_format": "%(name)-20s\t%(threadName)s\t%(levelname)-8s\t%(message)s", "master_eligibility": True, diff --git a/karapace/karapace_all.py b/karapace/karapace_all.py index 240da1008..1956a375a 100644 --- a/karapace/karapace_all.py +++ b/karapace/karapace_all.py @@ -2,10 +2,12 @@ Copyright (c) 2023 Aiven Ltd See LICENSE for details """ +from __future__ import annotations + from aiohttp.web_log import AccessLogger from contextlib import closing from karapace import version as karapace_version -from karapace.config import read_config +from karapace.config import Config, read_config from karapace.instrumentation.prometheus import PrometheusInstrumentation from karapace.kafka_rest_apis import KafkaRest from karapace.rapu import RestApp @@ -21,6 +23,37 @@ class KarapaceAll(KafkaRest, KarapaceSchemaRegistryController): pass +def _configure_logging(*, config: Config) -> None: + log_level = config.get("log_level", "DEBUG") + log_format = config.get("log_format", "%(name)-20s\t%(threadName)s\t%(levelname)-8s\t%(message)s") + + root_handler: logging.Handler | None = None + log_handler = config.get("log_handler", None) + if "systemd" == log_handler: + from systemd import journal + + root_handler = journal.JournalHandler(SYSLOG_IDENTIFIER="karapace") + if "stdout" == log_handler or log_handler is None: + root_handler = logging.StreamHandler(stream=sys.stdout) + else: + logging.basicConfig(level=logging.INFO, format=log_format) + logging.getLogger().setLevel(log_level) + + if root_handler is not None: + root_handler.setFormatter(logging.Formatter(log_format)) + root_handler.setLevel(log_level) + root_handler.set_name(name="karapace") + logging.root.addHandler(root_handler) + + logging.root.setLevel(log_level) + + if config.get("access_logs_debug") is True: + config["access_log_class"] = DebugAccessLogger + logging.getLogger("aiohttp.access").setLevel(logging.DEBUG) + else: + config["access_log_class"] = AccessLogger + + def main() -> int: parser = argparse.ArgumentParser(prog="karapace", description="Karapace: Your Kafka essentials in one tool") parser.add_argument("--version", action="version", help="show program version", version=karapace_version.__version__) @@ -30,13 +63,7 @@ def main() -> int: with closing(arg.config_file): config = read_config(arg.config_file) - logging.basicConfig(level=logging.INFO, format=config["log_format"]) - logging.getLogger().setLevel(config["log_level"]) - if config.get("access_logs_debug") is True: - config["access_log_class"] = DebugAccessLogger - logging.getLogger("aiohttp.access").setLevel(logging.DEBUG) - else: - config["access_log_class"] = AccessLogger + _configure_logging(config=config) app: RestApp if config["karapace_rest"] and config["karapace_registry"]: diff --git a/mypy.ini b/mypy.ini index 15ab9042f..1c51fffb1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -85,3 +85,6 @@ ignore_missing_imports = True [mypy-networkx.*] ignore_missing_imports = True + +[mypy.systemd.*] +ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index f29c94217..2cb0c0c66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ Issues = "https://github.com/Aiven-Open/karapace/issues" [project.optional-dependencies] sentry-sdk = ["sentry-sdk>=1.6.0"] +systemd-logging = ["systemd-python==235"] ujson = ["ujson"] dev = [ # Developer QoL diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 64acaaa4b..153980a37 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -97,6 +97,8 @@ six==1.16.0 # python-dateutil sniffio==1.3.1 # via anyio +systemd-python[systemd-logging]==235 + # via -r requirements.in tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) typing-extensions==4.12.2