diff --git a/Cargo.lock b/Cargo.lock index f6c4ab8..305d076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -624,7 +624,7 @@ dependencies = [ ] [[package]] -name = "kcl" +name = "kommitted" version = "0.1.0" dependencies = [ "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 4514827..9507447 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "kcl" +name = "kommitted" version = "0.1.0" edition = "2021" authors = [ @@ -7,8 +7,8 @@ authors = [ "Ivan De Marino ", "Kafkesc ", ] -repository = "https://github.com/kafkesc/kcl" -homepage = "https://github.com/kafkesc/kcl" +repository = "https://github.com/kafkesc/kommitted" +homepage = "https://github.com/kafkesc/kommitted" description = "Measure Kafka Consumer Offset Lag and Time Lag" license = "MIT OR Apache-2.0" keywords = ["kafka", "consume", "lag", "monitoring", "performance"] diff --git a/METRICS.md b/METRICS.md index 3b2a6b7..dd8a970 100644 --- a/METRICS.md +++ b/METRICS.md @@ -1,11 +1,11 @@ -# Metrics exposed by KCL +# Metrics exposed by Kommitted ## Metrics ### Consumer Metrics
-
kcl_kafka_consumer_partition_lag_milliseconds
+
kommitted_kafka_consumer_partition_lag_milliseconds
Description: The time difference (time lag) between when the latest offset was produced and the latest consumed offset was consumed, by the consumer of the topic partition, expressed in milliseconds. NOTE: '-1, -1' means 'unknown'.
Labels: cluster_id, group, topic, partition, member_id, member_host, member_client_id
@@ -15,7 +15,7 @@
-
kcl_kafka_consumer_partition_lag_offset
+
kommitted_kafka_consumer_partition_lag_offset
Description: The difference (lag) between the last produced offset and the last consumed offset, by the consumer of the topic partition. NOTE: '0, -1' means 'unknown'.
Labels: cluster_id, group, topic, partition, member_id, member_host, member_client_id
@@ -25,7 +25,7 @@
-
kcl_kafka_consumer_partition_offset
+
kommitted_kafka_consumer_partition_offset
Description: The last consumed offset by the consumer of the topic partition. NOTE: '0, -1' means 'unknown'.
Labels: cluster_id, group, topic, partition, member_id, member_host, member_client_id
@@ -37,7 +37,7 @@ ### Topic Partition Metrics
-
kcl_kafka_partition_earliest_available_offset
+
kommitted_kafka_partition_earliest_available_offset
Description: Earliest offset available to consumers of the topic partition.
Labels: cluster_id, topic, partition
@@ -47,7 +47,7 @@
-
kcl_kafka_partition_latest_available_offset
+
kommitted_kafka_partition_latest_available_offset
Description: Latest offset available to consumers of the topic partition.
Labels: cluster_id, topic, partition
@@ -59,7 +59,7 @@ ### Topic Partition Offset Tracking Metrics
-
kcl_kafka_partition_earliest_tracked_offset
+
kommitted_kafka_partition_earliest_tracked_offset
Description: Earliest offset tracked to estimate the lag of consumers of the topic partition.
Labels: cluster_id, topic, partition
@@ -69,7 +69,7 @@
-
kcl_kafka_partition_latest_tracked_offset
+
kommitted_kafka_partition_latest_tracked_offset
Description: Latest offset tracked to estimate the lag of consumers of the topic partition.
Labels: cluster_id, topic, partition
diff --git a/README.md b/README.md index 9a63e24..e463e6f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# KCL - Kafka Consumer Lag +# Kommitted - Measure Kafka consumers lag
@@ -6,13 +6,12 @@ Measure Kafka Consumer **Offset Lag** _and_ **Time Lag**
-**KCL** is a service to measure the _Lag_ (i.e. _Latency_) of Kafka consumers. -It works with all consumers that _commit_ their offsets into Kafka the -[standard way](https://kafka.apache.org/documentation/#design_consumerposition), as it +**Kommitted** is a service to measure the _Lag_ (i.e. _Latency_) of Kafka consumers. +It works with all consumers that _commit_ their offsets into Kafka (i.e. the +[standard way](https://kafka.apache.org/documentation/#design_consumerposition)), as it consumes the internal `__consumer_offsets` topic. -Metrics are exported (at the `/metrics` endpoint, following the -[Prometheus](https://prometheus.io/) +Metrics are exported following the [Prometheus](https://prometheus.io/) [Exposition formats](https://prometheus.io/docs/instrumenting/exposition_formats/#exposition-formats). Please see [DESIGN.md](./DESIGN.md) for details about the overall architecture, dependencies and other details. @@ -25,8 +24,8 @@ Please see [DESIGN.md](./DESIGN.md) for details about the overall architecture, * [x] Offset and Lag metrics are tracked with all contextual information to identify exact topic partition assignments * [ ] Exposes additional metrics to track status of Kafka cluster (topics, members, brokers, partitions) * [ ] Exposes Kafka-polling metrics, to assess its own performance -* [x] Metrics exposed in [Prometheus format](https://prometheus.io/docs/instrumenting/exposition_formats/#exposition-formats) -* [ ] REST API to build further automation on top of it (eg. auto-scaling logics that depend on Consumer Group lag) +* [x] Metrics exposed in [Prometheus format](https://prometheus.io/docs/instrumenting/exposition_formats/#exposition-formats), at `/metrics` endpoint +* [ ] REST API to build further automation on top of it (e.g. auto-scaling logics that depend on Consumer Group lag) All of this comes based on: @@ -45,18 +44,19 @@ WIP ## Usage -KCL supports _compact_ (`-h`) and _extended_ (`--help`) usage instructions. Use the former for a quick look up; -the latter to better understand what each argument can do. +Kommitted supports _compact_ (`-h`) and _extended_ (`--help`) usage instructions. +Use the former for a quick look up; use the latter to better understand what +each argument can do.
- Compact: `kcl -h` + Compact: `kommitted -h` ```shell - Usage: kcl [OPTIONS] --brokers + Usage: kommitted [OPTIONS] --brokers Options: -b, --brokers Initial Kafka Brokers to connect to (format: 'HOST:PORT,...') - --client-id Client identifier used by the internal Kafka (Admin) Client [default: kcl] + --client-id Client identifier used by the internal Kafka (Admin) Client [default: kommitted] --kafka-conf Additional configuration used by the internal Kafka (Admin) Client (format: 'CONF_KEY:CONF_VAL'). --cluster-id Override identifier of the monitored Kafka Cluster --history For each Topic Partition, how much history of offsets to track in memory. [default: 3600] @@ -69,10 +69,10 @@ the latter to better understand what each argument can do.
- Extended: `kcl --help` + Extended: `kommitted --help` ```shell - Usage: kcl [OPTIONS] --brokers + Usage: kommitted [OPTIONS] --brokers Options: -b, --brokers @@ -85,7 +85,7 @@ the latter to better understand what each argument can do. Equivalent to '--config=client.id:my-client-id'. - [default: kcl] + [default: kommitted] --kafka-conf Additional configuration used by the internal Kafka (Admin) Client (format: 'CONF_KEY:CONF_VAL'). @@ -129,7 +129,7 @@ the latter to better understand what each argument can do. * '-vv' = 'DEBUG' * '-vvv' = 'TRACE' - Alternatively, set environment variable 'KCL_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. + Alternatively, set environment variable 'KOMMITTED_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. -q, --quiet... Quiet logging. @@ -138,7 +138,7 @@ the latter to better understand what each argument can do. * '-q' = 'ERROR' * '-qq' = 'OFF' - Alternatively, set environment variable 'KCL_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. + Alternatively, set environment variable 'KOMMITTED_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. -h, --help Print help (see a summary with '-h') @@ -151,7 +151,7 @@ the latter to better understand what each argument can do. ### Connect to Kafka cluster requiring [`SASL_SSL`](https://en.wikipedia.org/wiki/Simple_Authentication_and_Security_Layer) ```shell -$ kcl \ +$ kommitted \ --brokers {{ BOOTSTRAP_BROKERS or BROKER_ENDPOINT }} \ --config security.protocol:SASL_SSL \ --config sasl.mechanisms=PLAIN \ @@ -174,7 +174,7 @@ Ksunami follows the long tradition of `-v/-q` to control the verbosity of it's l | `-vvv...` | `TRACE` | | It uses [log](https://crates.io/crates/log) and [env_logger](https://crates.io/crates/env_logger), -and so logging can be configured and fine-tuned using the Environment Variable `KCL_LOG`. +and so logging can be configured and fine-tuned using the Environment Variable `KOMMITTED_LOG`. Please take a look at [env_logger doc](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) for more details. diff --git a/src/cli.rs b/src/cli.rs index 0efaa55..dbf57c6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -82,7 +82,7 @@ pub struct Cli { /// * '-vv' = 'DEBUG' /// * '-vvv' = 'TRACE' /// - /// Alternatively, set environment variable 'KCL_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. + /// Alternatively, set environment variable 'KOMMITTED_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. #[arg(short, long, action = clap::ArgAction::Count, verbatim_doc_comment)] pub verbose: u8, @@ -92,7 +92,7 @@ pub struct Cli { /// * '-q' = 'ERROR' /// * '-qq' = 'OFF' /// - /// Alternatively, set environment variable 'KCL_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. + /// Alternatively, set environment variable 'KOMMITTED_LOG=(ERROR|WARN|INFO|DEBUG|TRACE|OFF)'. #[arg(short, long, action = clap::ArgAction::Count, verbatim_doc_comment)] pub quiet: u8, } diff --git a/src/constants.rs b/src/constants.rs index 8c04986..cc13487 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -6,7 +6,8 @@ pub(crate) const KONSUMER_OFFSETS_DATA_TOPIC: &str = "__consumer_offsets"; /// This is the Consumer Group (`group.id`) value used by /// the Consumer inside the `konsumer_offsets_data` module. -pub(crate) const KONSUMER_OFFSETS_KCL_CONSUMER: &str = "__kcl__consumer_offsets_consumer"; +pub(crate) const KOMMITTED_CONSUMER_OFFSETS_CONSUMER: &str = + "__kommitted__consumer_offsets_consumer"; /// The default host to bind to when launching internal HTTP server. pub(crate) const DEFAULT_HTTP_HOST: &str = "localhost"; diff --git a/src/consumer_groups/emitter.rs b/src/consumer_groups/emitter.rs index ae44f67..5da1652 100644 --- a/src/consumer_groups/emitter.rs +++ b/src/consumer_groups/emitter.rs @@ -10,7 +10,7 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; -use crate::constants::KONSUMER_OFFSETS_KCL_CONSUMER; +use crate::constants::KOMMITTED_CONSUMER_OFFSETS_CONSUMER; use crate::internals::Emitter; use crate::kafka_types::{Group, GroupWithMembers, Member, MemberWithAssignment, TopicPartition}; @@ -35,7 +35,7 @@ impl From for ConsumerGroups { for g in gl.groups() { // Ignore own consumer of `__consumer_offsets` topic - if g.name() == KONSUMER_OFFSETS_KCL_CONSUMER { + if g.name() == KOMMITTED_CONSUMER_OFFSETS_CONSUMER { continue; } diff --git a/src/http/mod.rs b/src/http/mod.rs index 6c5c2af..5f3ac2e 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -216,32 +216,32 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR // // --- CLUSTER METRICS --- // - // TODO `kcl_consumer_groups_total` + // TODO `kommitted_consumer_groups_total` // LABELS: cluster_id? // - // TODO `kcl_consumer_group_members_total` + // TODO `kommitted_consumer_group_members_total` // LABELS: cluster_id? // - // TODO `kcl_cluster_status_brokers_total` + // TODO `kommitted_cluster_status_brokers_total` // LABELS: cluster_id? // - // TODO `kcl_cluster_status_topics_total` + // TODO `kommitted_cluster_status_topics_total` // LABELS: cluster_id? // - // TODO `kcl_cluster_status_partitions_total` + // TODO `kommitted_cluster_status_partitions_total` // LABELS: cluster_id? // - // --- KCL INTERNAL METRICS --- + // --- KOMMITTED INTERNAL METRICS --- // - // TODO `kcl_consumer_groups_poll_time_seconds` + // TODO `kommitted_consumer_groups_poll_time_seconds` // HELP: Time taken to fetch information about all consumer groups in the cluster. // LABELS: cluster_id? // - // TODO `kcl_cluster_status_poll_time_ms` + // TODO `kommitted_cluster_status_poll_time_ms` // HELP: Time taken to fetch information about the composition of the cluster (brokers, topics, partitions). // LABELS: cluster_id? // - // TODO `kcl_partitions_watermark_offsets_poll_time_ms` + // TODO `kommitted_partitions_watermark_offsets_poll_time_ms` // HELP: Time taken to fetch earliest/latest (watermark) offsets of all the topic partitions of the cluster. // LABELS: cluster_id? diff --git a/src/konsumer_offsets_data/emitter.rs b/src/konsumer_offsets_data/emitter.rs index a21d34c..2977ca9 100644 --- a/src/konsumer_offsets_data/emitter.rs +++ b/src/konsumer_offsets_data/emitter.rs @@ -10,7 +10,7 @@ use rdkafka::{ use tokio::{sync::mpsc, task::JoinHandle, time::Duration}; use tokio_util::sync::CancellationToken; -use crate::constants::{KONSUMER_OFFSETS_DATA_TOPIC, KONSUMER_OFFSETS_KCL_CONSUMER}; +use crate::constants::{KOMMITTED_CONSUMER_OFFSETS_CONSUMER, KONSUMER_OFFSETS_DATA_TOPIC}; use crate::internals::Emitter; const CHANNEL_SIZE: usize = 2000; @@ -43,7 +43,7 @@ impl KonsumerOffsetsDataEmitter { client_config.set("enable.partition.eof", "false"); if client_config.get("group.id").is_none() { - client_config.set("group.id", KONSUMER_OFFSETS_KCL_CONSUMER); + client_config.set("group.id", KOMMITTED_CONSUMER_OFFSETS_CONSUMER); } client_config.set_log_level(RDKafkaLogLevel::Warning); diff --git a/src/lag_register/register.rs b/src/lag_register/register.rs index 8380577..b14a469 100644 --- a/src/lag_register/register.rs +++ b/src/lag_register/register.rs @@ -9,7 +9,7 @@ use konsumer_offsets::{GroupMetadata, KonsumerOffsetsData, OffsetCommit}; use log::Level::Trace; use tokio::sync::{mpsc, RwLock}; -use crate::constants::KONSUMER_OFFSETS_KCL_CONSUMER; +use crate::constants::KOMMITTED_CONSUMER_OFFSETS_CONSUMER; use crate::consumer_groups::ConsumerGroups; use crate::internals::Awaitable; use crate::kafka_types::{Group, Member, TopicPartition}; @@ -130,7 +130,7 @@ async fn process_consumer_groups( ) { for (group_name, group_with_members) in cg.groups.into_iter() { // Ignore own consumer of `__consumer_offsets` topic. - if group_name == KONSUMER_OFFSETS_KCL_CONSUMER { + if group_name == KOMMITTED_CONSUMER_OFFSETS_CONSUMER { continue; } @@ -203,7 +203,7 @@ async fn process_offset_commit( po_reg: Arc, ) { // Ignore own consumer of `__consumer_offsets` topic. - if oc.group == KONSUMER_OFFSETS_KCL_CONSUMER { + if oc.group == KOMMITTED_CONSUMER_OFFSETS_CONSUMER { return; } @@ -268,7 +268,7 @@ async fn process_group_metadata( lag_register_groups: Arc>>, ) { // Ignore own consumer of `__consumer_offsets` topic. - if gm.group == KONSUMER_OFFSETS_KCL_CONSUMER { + if gm.group == KOMMITTED_CONSUMER_OFFSETS_CONSUMER { return; } diff --git a/src/logging.rs b/src/logging.rs index 221de6f..12fea35 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -1,8 +1,8 @@ -pub const LOG_FILTER_ENV_VAR: &str = "KCL_LOG"; +pub const LOG_FILTER_ENV_VAR: &str = "KOMMITTED_LOG"; /// Log level will be configured based on the given `verbosity_level`. /// -/// If the env var `KCL_LOG` is set, that will take precedence and configuration +/// If the env var `KOMMITTED_LOG` is set, that will take precedence and configuration /// will be based on the rules described [here](https://docs.rs/env_logger/latest/env_logger/#enabling-logging). pub fn init(verbosity_level: i8) { let default_log_level = match verbosity_level { diff --git a/src/prometheus_metrics/mod.rs b/src/prometheus_metrics/mod.rs index b70f729..ceb0069 100644 --- a/src/prometheus_metrics/mod.rs +++ b/src/prometheus_metrics/mod.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use prometheus::Registry; -pub const NAMESPACE: &str = "kcl"; +pub const NAMESPACE: &str = "kommitted"; pub const LABEL_CLUSTER_ID: &str = "cluster_id"; pub const LABEL_GROUP: &str = "group";