diff --git a/conf/input.elasticsearch/elasticsearch.toml b/conf/input.elasticsearch/elasticsearch.toml index 225b636f..3966a649 100644 --- a/conf/input.elasticsearch/elasticsearch.toml +++ b/conf/input.elasticsearch/elasticsearch.toml @@ -10,15 +10,30 @@ # append some labels to metrics # labels = { cluster="cloud-n9e-es" } +# either /_nodes/stats or /_nodes/_local/stats depending on this setting +local = false + ## specify a list of one or more Elasticsearch servers # servers = ["http://localhost:9200"] servers = [] +## HTTP Basic Authentication username and password. +username = "elastic" +password = "password" + +## API key for authentication +# api_key = "your_api_key" + ## Timeout for HTTP requests to the elastic search server(s) http_timeout = "10s" -# either /_nodes/stats or /_nodes/_local/stats depending on this setting -local = false +## all_nodes If true, query stats for all nodes in the cluster, rather than just the node we connect to. +all_nodes = true + +## node_stats is a list of sub-stats that you want to have gathered. Valid options +## are "indices", "os", "process", "jvm", "thread_pool", "fs", "transport", "http", +## "breaker". Per default, all stats are gathered. +node_stats = ["jvm", "breaker", "process", "os", "fs", "indices", "thread_pool", "transport"] ## Set cluster_health to true when you want to obtain cluster health stats cluster_health = true @@ -27,7 +42,7 @@ cluster_health = true ## The options are ## - indices (default) ## - cluster -cluster_health_level = "cluster" +cluster_health_level = "indices" ## Set cluster_stats to true when you want to obtain cluster stats. cluster_stats = true @@ -36,17 +51,44 @@ cluster_stats = true ## Use of wildcards is allowed. Use a wildcard at the end to retrieve index names that end with a changing value, like a date. # indices_include = ["zipkin*"] -## use "shards" or blank string for indices level -indices_level = "" +## Exclude indices from the list of indices to collect. If true, query stats for all indices in the cluster. +export_indices = false -## node_stats is a list of sub-stats that you want to have gathered. Valid options -## are "indices", "os", "process", "jvm", "thread_pool", "fs", "transport", "http", -## "breaker". Per default, all stats are gathered. -node_stats = ["jvm", "breaker", "process", "os", "fs", "indices", "thread_pool", "transport"] +## Export indices settings. If true, query settings stats for all indices in the cluster. +export_indices_settings = false -## HTTP Basic Authentication username and password. -username = "elastic" -password = "password" +## Export indices mappings. If true, query mappings stats for all indices in the cluster. +export_indices_mappings = false + +## Export indices aliases. If true, query aliases stats for all indices in the cluster. +export_indices_aliases = false + +## Export index lifecycle politics for indices in the cluster. +export_ilm = false + +## If true, query stats for all indices in the cluster, including shard-level stats (implies `es.indices=true`). +export_shards = false + +## If true, query stats for SLM. +export_slm = false + +## If true, query stats for data streams. +export_data_stream = false + +## If true, query stats for snapshots. +export_snapshots = false + +## Export cluster settings. If true, query settings stats for the cluster. +export_cluster_settings = false + +# Cluster info update interval for the cluster label (default: 5m) +cluster_info_interval = "5m" + +# Region for AWS elasticsearch +# aws_region = "" + +# Role ARN of an IAM role to assume. +# aws_role_arn = "" ## Optional TLS Config # use_tls = false @@ -60,4 +102,4 @@ password = "password" ## Each 'indices_include' entry ending with a wildcard (*) or glob matching pattern will group together all indices that match it, and ## sort them by the date or number after the wildcard. Metrics then are gathered for only the 'num_most_recent_indices' amount of most ## recent indices. -num_most_recent_indices = 1 \ No newline at end of file +num_most_recent_indices = 1 diff --git a/go.mod b/go.mod index 747fb02c..e1b3903a 100644 --- a/go.mod +++ b/go.mod @@ -110,6 +110,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tjfoc/gmsm v1.3.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect golang.org/x/arch v0.3.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect @@ -122,6 +123,7 @@ require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/NVIDIA/go-dcgm v0.0.0-20240118201113-3385e277e49f github.com/NVIDIA/go-nvml v0.12.0-2 + github.com/alecthomas/kingpin/v2 v2.4.0 github.com/alibabacloud-go/cms-20190101/v8 v8.0.0 github.com/alibabacloud-go/cms-export-20211101/v2 v2.0.0 github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.0 @@ -135,6 +137,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 github.com/beevik/ntp v1.3.1 github.com/bits-and-blooms/bitset v1.13.0 + github.com/blang/semver/v4 v4.0.0 github.com/bmatcuk/doublestar/v3 v3.0.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/dennwc/btrfs v0.0.0-20230312211831-a1f570bd01a1 @@ -260,7 +263,7 @@ require ( github.com/hashicorp/nomad/api v0.0.0-20221102143410-8a95f1239005 // indirect github.com/hashicorp/serf v0.9.7 // indirect github.com/hetznercloud/hcloud-go v1.35.3 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.12 github.com/ionos-cloud/sdk-go/v6 v6.1.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 6aab74a2..2caf5d18 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,8 @@ github.com/Shopify/sarama v1.36.0/go.mod h1:9glG3eX83tgVYJ5aVtrjVUnEsOPqQIBGx1BW github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64= github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg= github.com/alecthomas/go-thrift v0.0.0-20170109061633-7914173639b2/go.mod h1:CxCgO+NdpMdi9SsTlGbc0W+/UNxO3I0AabOEJZ3w61w= +github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/kong v0.2.1/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= github.com/alecthomas/participle v0.4.1 h1:P2PJWzwrSpuCWXKnzqvw0b0phSfH1kJo4p2HvLynVsI= github.com/alecthomas/participle v0.4.1/go.mod h1:T8u4bQOSMwrkTWOSyt8/jSFPEnRtd0FKFMjVfYBlqPs= @@ -193,6 +195,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI= github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@ -1063,6 +1067,8 @@ github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= diff --git a/inputs/elasticsearch/README.md b/inputs/elasticsearch/README.md index ccf46d2a..ded28664 100644 --- a/inputs/elasticsearch/README.md +++ b/inputs/elasticsearch/README.md @@ -1,14 +1,485 @@ -# elasticsearch +# Elasticsearch -forked from telegraf/elasticsearch +#### Elasticsearch 7.x 权限 -## 改动 +用户名和密码可以直接通过URI传递，或者通过ES_USERNAME和ES_PASSWORD环境变量传递。指定这两个环境变量将覆盖在URI中传递的认证信息（如果有的话）。 -- 不再处理json中的数组类型 -- 修改一些不合法的metric名称 -- 配置去掉cluster_stats_only_from_master -- 调整默认配置，不采集每个索引的监控数据，nodestats不采集http数据 +ES 7.x 支持基于角色的访问控制（RBACs）。`elasticsearch` 插件需要以下安全权限： -## 监控大盘 +| 设置 | 所需权限 | 描述 | +|:------------------------|:-----------------------------------------------------------------|:--------------------------------------------------------------------------------------| +| export_cluster_settings | `cluster` `monitor` | | +| exporter defaults | `cluster` `monitor` | 包括所有集群只读操作，如集群健康与状态、热点线程、节点信息、节点和集群统计信息以及待处理的集群任务。 | +| export_indices | `indices` `monitor` (每个索引或 `*`) | 所有监控所需的操作（恢复、段信息、索引统计和状态） | +| export_indices_settings | `indices` `monitor` (每个索引或 `*`) | | +| export_indices_mappings | `indices` `view_index_metadata` (每个索引或 `*`) | | +| export_shards | 不确定是 `indices` 或 `cluster` `monitor` 或两者都需要 | | +| export_snapshots | `cluster:admin/snapshot/status` 和 `cluster:admin/repository/get` | [ES 论坛帖子](https://discuss.elastic.co/t/permissions-for-backup-user-with-x-pack/88057) | +| export_slm | `read_slm` | | +| export_data_stream | `monitor` 或 `manage` (每个索引或 `*`) | | -在该 README 文件同级目录下的 dashboard.json 可以直接导入夜莺使用 \ No newline at end of file +### 与旧版`elastisearch`插件的区别 + +- `elasticsearch_cluster_health_active_shards_percent_as_number`改为`elasticsearch_cluster_health_active_shards_percent`。 +- `elasticsearch_cluster_health_status`和`elasticsearch_cluster_health_status_code`合并为`elasticsearch_cluster_health_status`，值为`green=1`、`yellow=2`、`red=3`。 +- `elasticsearch_process_cpu_total_in_millis`改为`elasticsearch_process_cpu_seconds_total`，单位为秒。 +- `elasticsearch_jvm_uptime_in_millis`改为`elasticsearch_jvm_uptime_seconds`，单位为秒。以此类推，所有`*_in_millis`的指标都改为`*_seconds`。 + +### Metrics + +#### `cluster_health = true` + +| 名称 | 类型 | 描述 | +|-----------------------------------------------------------------|------------|--------------------------| +| `elasticsearch_cluster_health_active_primary_shards` | GaugeValue | 集群中主分片的数量。这是跨所有索引的聚合总数。 | +| `elasticsearch_cluster_health_active_shards` | GaugeValue | 所有索引中所有分片的聚合总数，包括副本分片。 | +| `elasticsearch_cluster_health_active_shards_percent` | GaugeValue | 集群中活跃分片的百分比。 | +| `elasticsearch_cluster_health_delayed_unassigned_shards` | GaugeValue | 为减少重新分配开销而延迟的分片数量。 | +| `elasticsearch_cluster_health_initializing_shards` | GaugeValue | 正在新创建的分片计数。 | +| `elasticsearch_cluster_health_number_of_data_nodes` | GaugeValue | 集群中数据节点的数量。 | +| `elasticsearch_cluster_health_number_of_in_flight_fetch` | GaugeValue | 正在进行的分片信息请求的数量。 | +| `elasticsearch_cluster_health_task_max_waiting_in_queue_millis` | GaugeValue | 任务在队列中等待的最长时间（毫秒）。 | +| `elasticsearch_cluster_health_number_of_nodes` | GaugeValue | 集群中节点的数量。 | +| `elasticsearch_cluster_health_number_of_pending_tasks` | GaugeValue | 尚未执行的集群级别变更的数量。 | +| `elasticsearch_cluster_health_relocating_shards` | GaugeValue | 当前从一个节点移动到另一个节点的分片数量。 | +| `elasticsearch_cluster_health_unassigned_shards` | GaugeValue | 存在于集群状态中但在集群本身中找不到的分片数量。 | + +#### `cluster_health = true` 和 `cluster_health_level = "indices"` + +| 名称 | 类型 | 描述 | +|--------------------------------------------------------------|------------|--------------------------| +| `elasticsearch_cluster_health_indices_active_primary_shards` | GaugeValue | 集群中主分片的数量。这是跨所有索引的聚合总数。 | +| `elasticsearch_cluster_health_indices_active_shards` | GaugeValue | 所有索引中所有分片的聚合总数，包括副本分片。 | +| `elasticsearch_cluster_health_indices_initializing_shards` | GaugeValue | 正在新创建的分片计数。 | +| `elasticsearch_cluster_health_indices_number_of_replicas` | GaugeValue | 集群中副本的数量。 | +| `elasticsearch_cluster_health_indices_number_of_shards` | GaugeValue | 集群中分片的数量。 | +| `elasticsearch_cluster_health_indices_relocating_shards` | GaugeValue | 当前从一个节点移动到另一个节点的分片数量。 | +| `elasticsearch_cluster_health_indices_unassigned_shards` | GaugeValue | 存在于集群状态中但在集群本身中找不到的分片数量。 | + +#### `export_cluster_settings = true` + +| 名称 | 类型 | 描述 | +|------------------------------------------------------------------------|------------|---------------------| +| `elasticsearch_clustersettings_stats_shard_allocation_enabled` | GaugeValue | 集群范围的分片路由分配设置的当前模式。 | +| `elasticsearch_clustersettings_stats_max_shards_per_node` | GaugeValue | 每个节点的最大分片数设置的当前值。 | +| `elasticsearch_clustersettings_allocation_threshold_enabled` | GaugeValue | 磁盘分配决策器是否启用。 | +| `elasticsearch_clustersettings_allocation_watermark_flood_stage_ratio` | GaugeValue | 作为比例的洪水阶段水位标记。 | +| `elasticsearch_clustersettings_allocation_watermark_high_ratio` | GaugeValue | 作为比例的磁盘使用的高水位标记。 | +| `elasticsearch_clustersettings_allocation_watermark_low_ratio` | GaugeValue | 作为比例的磁盘使用的低水位标记。 | +| `elasticsearch_clustersettings_allocation_watermark_flood_stage_bytes` | GaugeValue | 以字节为单位的洪水阶段水位标记。 | +| `elasticsearch_clustersettings_allocation_watermark_high_bytes` | GaugeValue | 以字节为单位的磁盘使用的高水位标记。 | +| `elasticsearch_clustersettings_allocation_watermark_low_bytes` | GaugeValue | 以字节为单位的磁盘使用的低水位标记。 | + +#### `cluster_stats = true` + +| 名称 | 类型 | 描述 | +|-----------------------------------------------------------------------------|--------------|------------------| +| `elasticsearch_clusterstats_indices_count` | CounterValue | 完成的字节数 | +| `elasticsearch_clusterstats_indices_completion_size_in_bytes` | CounterValue | 完成的字节数 | +| `elasticsearch_clusterstats_indices_docs_count` | GaugeValue | 此集群上的文档计数 | +| `elasticsearch_clusterstats_indices_docs_deleted` | GaugeValue | 此集群上已删除的文档计数 | +| `elasticsearch_clusterstats_indices_fielddata_evictions` | CounterValue | 字段数据的驱逐次数 | +| `elasticsearch_clusterstats_indices_fielddata_memory_size_in_bytes` | GaugeValue | 字段数据缓存使用的内存量（字节） | +| `elasticsearch_clusterstats_indices_query_cache_cache_count` | CounterValue | 查询缓存的缓存计数 | +| `elasticsearch_clusterstats_indices_query_cache_cache_size` | GaugeValue | 查询缓存的缓存大小 | +| `elasticsearch_clusterstats_indices_query_cache_evictions` | CounterValue | 查询缓存的驱逐次数 | +| `elasticsearch_clusterstats_indices_query_cache_hit_count` | CounterValue | 查询缓存命中计数 | +| `elasticsearch_clusterstats_indices_query_cache_memory_size_in_bytes` | GaugeValue | 查询缓存内存使用量（字节） | +| `elasticsearch_clusterstats_indices_query_cache_miss_count` | CounterValue | 查询缓存未命中计数 | +| `elasticsearch_clusterstats_indices_query_cache_total_count` | CounterValue | 查询缓存总计数 | +| `elasticsearch_clusterstats_indices_segments_count` | GaugeValue | 集群索引段计数 | +| `elasticsearch_clusterstats_indices_segments_doc_values_memory_in_bytes` | GaugeValue | 文档值内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_fixed_bit_set_memory_in_bytes` | GaugeValue | 固定位集内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_index_writer_memory_in_bytes` | GaugeValue | 索引编写器内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_max_unsafe_auto_id_timestamp` | GaugeValue | 索引编写器内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_memory_in_bytes` | GaugeValue | 段当前内存大小（字节） | +| `elasticsearch_clusterstats_indices_segments_norms_memory_in_bytes` | GaugeValue | 标准化内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_points_memory_in_bytes` | GaugeValue | 点值内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_stored_fields_memory_in_bytes` | GaugeValue | 存储字段内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_term_vectors_memory_in_bytes` | GaugeValue | 术语向量内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_terms_memory_in_bytes` | GaugeValue | 术语内存使用量（字节） | +| `elasticsearch_clusterstats_indices_segments_version_map_memory_in_bytes` | GaugeValue | 版本映射内存使用量（字节） | +| `elasticsearch_clusterstats_indices_shards_total` | GaugeValue | 集群中的总分片数 | +| `elasticsearch_clusterstats_indices_shards_replication` | GaugeValue | 分片复制数 | +| `elasticsearch_clusterstats_indices_shards_primaries` | GaugeValue | 集群中的主分片数 | +| `elasticsearch_clusterstats_indices_shards_index_primaries_avg` | GaugeValue | 每个索引的平均主分片数 | +| `elasticsearch_clusterstats_indices_shards_index_primaries_max` | GaugeValue | 每个索引的最大主分片数 | +| `elasticsearch_clusterstats_indices_shards_index_primaries_min` | GaugeValue | 每个索引的最小主分片数 | +| `elasticsearch_clusterstats_indices_shards_index_replication_avg` | GaugeValue | 每个索引的平均复制分片数 | +| `elasticsearch_clusterstats_indices_shards_index_replication_max` | GaugeValue | 每个索引的最大复制分片数 | +| `elasticsearch_clusterstats_indices_shards_index_replication_min` | GaugeValue | 每个索引的最小复制分片数 | +| `elasticsearch_clusterstats_indices_shards_index_shards_avg` | GaugeValue | 每个索引的平均分片数 | +| `elasticsearch_clusterstats_indices_shards_index_shards_max` | GaugeValue | 每个索引的最大分片数 | +| `elasticsearch_clusterstats_indices_shards_index_shards_min` | GaugeValue | 每个索引的最小分片数 | +| `elasticsearch_clusterstats_indices_store_size_in_bytes` | GaugeValue | 当前存储大小（字节） | +| `elasticsearch_clusterstats_indices_total_data_set_size_in_bytes` | GaugeValue | 数据集总大小（字节） | +| `elasticsearch_clusterstats_indices_reserved_in_bytes` | GaugeValue | 保留大小（字节） | +| `elasticsearch_clusterstats_nodes_count_coordinating_only` | GaugeValue | 仅协调节点计数 | +| `elasticsearch_clusterstats_nodes_count_data` | GaugeValue | 数据节点计数 | +| `elasticsearch_clusterstats_nodes_count_ingest` | GaugeValue | 摄取节点计数 | +| `elasticsearch_clusterstats_nodes_count_master` | GaugeValue | 主节点计数 | +| `elasticsearch_clusterstats_nodes_count_total` | GaugeValue | 集群中节点总计数 | +| `elasticsearch_clusterstats_nodes_fs_available_in_bytes` | GaugeValue | 可用磁盘空间（字节） | +| `elasticsearch_clusterstats_nodes_fs_free_in_bytes` | GaugeValue | 空闲磁盘空间（字节） | +| `elasticsearch_clusterstats_nodes_fs_total_in_bytes` | GaugeValue | 磁盘总空间（字节） | +| `elasticsearch_clusterstats_nodes_jvm_max_uptime_in_millis` | GaugeValue | JVM最大运行时间（毫秒） | +| `elasticsearch_clusterstats_nodes_jvm_mem_heap_max_in_bytes` | GaugeValue | 堆内存最大值（字节） | +| `elasticsearch_clusterstats_nodes_jvm_mem_heap_used_in_bytes` | GaugeValue | 使用的堆内存（字节） | +| `elasticsearch_clusterstats_nodes_jvm_threads` | GaugeValue | JVM线程数 | +| `elasticsearch_clusterstats_nodes_network_types_http_types_security4` | GaugeValue | HTTP安全4网络类型 | +| `elasticsearch_clusterstats_nodes_network_types_transport_types_security4` | GaugeValue | 传输安全4网络类型 | +| `elasticsearch_clusterstats_nodes_os_allocated_processors` | GaugeValue | 分配的处理器数 | +| `elasticsearch_clusterstats_nodes_os_available_processors` | GaugeValue | 可用处理器数 | +| `elasticsearch_clusterstats_nodes_os_mem_free_in_bytes` | GaugeValue | 空闲内存（字节） | +| `elasticsearch_clusterstats_nodes_os_mem_free_percent` | GaugeValue | 空闲内存百分比 | +| `elasticsearch_clusterstats_nodes_os_mem_total_in_bytes` | GaugeValue | 总内存（字节） | +| `elasticsearch_clusterstats_nodes_os_mem_used_in_bytes` | GaugeValue | 使用的内存（字节） | +| `elasticsearch_clusterstats_nodes_os_mem_used_percent` | GaugeValue | 使用的内存百分比 | +| `elasticsearch_clusterstats_nodes_process_cpu_percent` | GaugeValue | 进程CPU使用百分比 | +| `elasticsearch_clusterstats_nodes_process_open_file_descriptors_avg` | GaugeValue | 打开的文件描述符平均数 | +| `elasticsearch_clusterstats_nodes_process_open_file_descriptors_max` | GaugeValue | 打开的文件描述符最大数 | +| `elasticsearch_clusterstats_nodes_process_open_file_descriptors_min` | GaugeValue | 打开的文件描述符最小数 | + +#### `export_data_stream = true` + +| 名称 | 类型 | 描述 | +|-------------------------------------------------------|--------------|--------------| +| `elasticsearch_data_stream_backing_indices_total` | CounterValue | 后备索引的数量 | +| `elasticsearch_data_stream_store_size_bytes` | CounterValue | 数据流的存储大小 | +| `elasticsearch_data_stream_stats_up` | gauge | 数据流收集的上行指标 | +| `elasticsearch_data_stream_stats_total_scrapes` | counter | 数据流统计的总抓取次数 | +| `elasticsearch_data_stream_stats_json_parse_failures` | counter | 数据流统计的解析失败次数 | + +#### `all_nodes = true` + +| 名称 | 类型 | 描述 | +|----------------------------------------------------------------|--------------|---------------------| +| `elasticsearch_os_cpu_load_average_1m` | GaugeValue | 短期负载平均值 | +| `elasticsearch_os_cpu_load_average_5m` | GaugeValue | 中期负载平均值 | +| `elasticsearch_os_cpu_load_average_15m` | GaugeValue | 长期负载平均值 | +| `elasticsearch_os_cpu_percent` | GaugeValue | 操作系统使用的CPU百分比 | +| `elasticsearch_os_mem_free_in_bytes` | GaugeValue | 可用物理内存量（字节） | +| `elasticsearch_os_mem_used_in_bytes` | GaugeValue | 已用物理内存量（字节） | +| `elasticsearch_os_mem_actual_free_in_bytes` | GaugeValue | 实际可用物理内存量（字节） | +| `elasticsearch_os_mem_actual_used_in_bytes` | GaugeValue | 实际已用物理内存量（字节） | +| `elasticsearch_os_mem_used_percent` | GaugeValue | 已用物理内存百分比 | +| `elasticsearch_os_mem_total_in_bytes` | GaugeValue | 物理内存总量（字节） | +| `elasticsearch_os_mem_free_percent` | GaugeValue | 可用物理内存百分比 | +| `elasticsearch_os_cgroup_cpu_cfs_period_micros` | GaugeValue | CPU CFS周期（微秒） | +| `elasticsearch_os_cgroup_cpu_cfs_quota_micros` | GaugeValue | CPU CFS配额（微秒） | +| `elasticsearch_os_cgroup_cpu_stat_number_of_elapsed_periods` | GaugeValue | CPU CFS配额中的时间段数量 | +| `elasticsearch_os_cgroup_cpu_stat_number_of_times_throttled` | GaugeValue | CPU CFS时间段被节流的次数 | +| `elasticsearch_os_cgroup_cpu_stat_time_throttled_nanos` | GaugeValue | CPU CFS被节流的时间（纳秒） | +| `elasticsearch_os_cgroup_cpuacct_usage_nanos` | GaugeValue | cpuacct使用时间（纳秒） | +| `elasticsearch_os_swap_used_in_bytes` | GaugeValue | 已用交换空间量（字节） | +| `elasticsearch_os_swap_total_in_bytes` | GaugeValue | 交换空间总量（字节） | +| `elasticsearch_os_swap_free_in_bytes` | GaugeValue | 可用交换空间量（字节） | +| `elasticsearch_indices_fielddata_memory_size_in_bytes` | GaugeValue | 字段数据缓存内存使用量（字节） | +| `elasticsearch_indices_fielddata_evictions` | CounterValue | 字段数据缓存逐出次数 | +| `elasticsearch_indices_completion_size_in_bytes` | CounterValue | 自动完成数据大小（字节） | +| `elasticsearch_indices_filter_cache_memory_size_in_bytes` | GaugeValue | 过滤器缓存内存使用量（字节） | +| `elasticsearch_indices_filter_cache_evictions` | CounterValue | 过滤器缓存逐出次数 | +| `elasticsearch_indices_query_cache_memory_size_in_bytes` | GaugeValue | 查询缓存内存使用量（字节） | +| `elasticsearch_indices_query_cache_evictions` | CounterValue | 查询缓存逐出次数 | +| `elasticsearch_indices_query_cache_total_count` | CounterValue | 查询缓存总计数 | +| `elasticsearch_indices_query_cache_cache_size` | GaugeValue | 查询缓存大小 | +| `elasticsearch_indices_query_cache_cache_count` | CounterValue | 查询缓存缓存计数 | +| `elasticsearch_indices_query_cache_hit_count` | CounterValue | 查询缓存命中计数 | +| `elasticsearch_indices_query_cache_miss_count` | CounterValue | 查询缓存未命中计数 | +| `elasticsearch_indices_request_cache_memory_size_in_bytes` | GaugeValue | 请求缓存内存使用量（字节） | +| `elasticsearch_indices_request_cache_evictions` | CounterValue | 请求缓存逐出次数 | +| `elasticsearch_indices_request_cache_hit_count` | CounterValue | 请求缓存命中计数 | +| `elasticsearch_indices_request_cache_miss_count` | CounterValue | 请求缓存未命中计数 | +| `elasticsearch_indices_translog_operations` | CounterValue | 总事务日志操作数 | +| `elasticsearch_indices_translog_size_in_bytes` | GaugeValue | 事务日志总大小（字节） | +| `elasticsearch_indices_get_time_seconds` | CounterValue | 获取操作总耗时（秒） | +| `elasticsearch_indices_get_total` | CounterValue | 总获取操作数 | +| `elasticsearch_indices_get_missing_time_seconds` | CounterValue | 缺失文档获取操作总耗时（秒） | +| `elasticsearch_indices_get_missing_total` | CounterValue | 缺失文档获取操作总数 | +| `elasticsearch_indices_get_exists_time_seconds` | CounterValue | 存在文档获取操作总耗时（秒） | +| `elasticsearch_indices_get_exists_total` | CounterValue | 存在文档获取操作总数 | +| `elasticsearch_indices_refresh_time_seconds_total` | CounterValue | 刷新操作总耗时（秒） | +| `elasticsearch_indices_refresh_total` | CounterValue | 总刷新操作数 | +| `elasticsearch_indices_search_query_time_seconds` | CounterValue | 搜索查询总耗时（秒） | +| `elasticsearch_indices_search_query_total` | CounterValue | 总搜索查询数 | +| `elasticsearch_indices_search_fetch_time_seconds` | CounterValue | 搜索抓取总耗时（秒） | +| `elasticsearch_indices_search_fetch_total` | CounterValue | 总搜索抓取数 | +| `elasticsearch_indices_search_suggest_total` | CounterValue | 总搜索建议数 | +| `elasticsearch_indices_search_suggest_time_seconds` | CounterValue | 搜索建议总耗时（秒） | +| `elasticsearch_indices_search_scroll_total` | CounterValue | 总滚动搜索数 | +| `elasticsearch_indices_search_scroll_time_seconds` | CounterValue | 滚动搜索总耗时（秒） | +| `elasticsearch_indices_docs_count` | GaugeValue | 该节点上的文档总数 | +| `elasticsearch_indices_docs_deleted` | GaugeValue | 该节点上被删除的文档数 | +| `elasticsearch_indices_store_size_in_bytes` | GaugeValue | 存储的索引数据总大小（字节） | +| `elasticsearch_indices_store_throttle_time_seconds_total` | CounterValue | 索引存储节流时间总计（秒） | +| `elasticsearch_indices_segments_memory_in_bytes` | GaugeValue | 当前段内存大小（字节） | +| `elasticsearch_indices_segments_count` | GaugeValue | 该节点上的索引段总数 | +| `elasticsearch_indices_segments_terms_memory_in_bytes` | GaugeValue | 该节点上术语的内存使用量（字节） | +| `elasticsearch_indices_segments_index_writer_memory_in_bytes` | GaugeValue | 该节点上索引编写器的内存使用量（字节） | +| `elasticsearch_indices_segments_max_unsafe_auto_id_timestamp` | GaugeValue | 该节点上最大不安全自动ID时间戳 | +| `elasticsearch_indices_segments_norms_memory_in_bytes` | GaugeValue | 该节点上标准化值的内存使用量（字节） | +| `elasticsearch_indices_segments_stored_fields_memory_in_bytes` | GaugeValue | 存储字段的内存使用量（字节） | +| `elasticsearch_indices_segments_doc_values_memory_in_bytes` | GaugeValue | 文档值的内存使用量（字节） | +| `elasticsearch_indices_segments_fixed_bit_set_memory_in_bytes` | GaugeValue | 固定位集的内存使用量（字节） | +| `elasticsearch_indices_segments_term_vectors_memory_in_bytes` | GaugeValue | 术语向量的内存使用量（字节） | +| `elasticsearch_indices_segments_points_memory_in_bytes` | GaugeValue | 点数据的内存使用量（字节） | +| `elasticsearch_indices_segments_version_map_memory_in_bytes` | GaugeValue | 版本映射的内存使用量（字节） | +| `elasticsearch_indices_flush_total` | CounterValue | 总刷新次数 | +| `elasticsearch_indices_flush_time_seconds` | CounterValue | 总刷新时间（秒） | +| `elasticsearch_indices_warmer_total` | CounterValue | 总预热次数 | +| `elasticsearch_indices_warmer_time_seconds_total` | CounterValue | 总预热时间（秒） | +| `elasticsearch_indices_indexing_index_time_seconds_total` | CounterValue | 索引时间总计（秒） | +| `elasticsearch_indices_indexing_index_total` | CounterValue | 总索引次数 | +| `elasticsearch_indices_indexing_delete_time_seconds_total` | CounterValue | 删除索引的总时间（秒） | +| `elasticsearch_indices_indexing_delete_total` | CounterValue | 总删除次数 | +| `elasticsearch_indices_indexing_is_throttled` | GaugeValue | 是否正在限制索引 | +| `elasticsearch_indices_indexing_throttle_time_seconds_total` | CounterValue | 索引限流时间总计（秒） | +| `elasticsearch_indices_merges_total` | CounterValue | 总合并次数 | +| `elasticsearch_indices_merges_current` | GaugeValue | 当前合并次数 | +| `elasticsearch_indices_merges_current_size_in_bytes` | GaugeValue | 当前合并大小（字节） | +| `elasticsearch_indices_merges_docs_total` | CounterValue | 文档合并总数 | +| `elasticsearch_indices_merges_total_size_in_bytes` | CounterValue | 合并总大小（字节） | +| `elasticsearch_indices_merges_total_time_seconds_total` | CounterValue | 合并总时间（秒） | +| `elasticsearch_indices_merges_total_throttled_time_seconds` | CounterValue | 合并被限流的总时间（秒） | +| `elasticsearch_jvm_threads_count` | GaugeValue | JVM线程数 | +| `elasticsearch_jvm_threads_peak_count` | GaugeValue | JVM线程峰值数 | +| `elasticsearch_jvm_timestamp` | GaugeValue | JVM时间戳 | +| `elasticsearch_jvm_mem_heap_used_in_bytes` | GaugeValue | JVM堆使用的内存量（字节） | +| `elasticsearch_jvm_mem_non_heap_used_in_bytes` | GaugeValue | JVM非堆使用的内存量（字节） | +| `elasticsearch_jvm_mem_heap_max_in_bytes` | GaugeValue | JVM堆最大内存量（字节） | +| `elasticsearch_jvm_mem_heap_used_percent` | GaugeValue | JVM堆使用的内存百分比 | +| `elasticsearch_jvm_mem_heap_committed_in_bytes` | GaugeValue | JVM堆提交的内存量（字节） | +| `elasticsearch_jvm_mem_non_heap_committed_in_bytes` | GaugeValue | JVM非堆提交的内存量（字节） | +| `elasticsearch_jvm_memory_pools_young_used_in_bytes` | GaugeValue | JVM年轻代使用的内存量（字节） | +| `elasticsearch_jvm_memory_pools_young_max_in_bytes` | CounterValue | JVM年轻代最大内存量（字节） | +| `elasticsearch_jvm_memory_pools_young_peak_used_in_bytes` | CounterValue | JVM年轻代峰值使用的内存量（字节） | +| `elasticsearch_jvm_memory_pools_young_peak_max_in_bytes` | CounterValue | JVM年轻代峰值最大内存量（字节） | +| `elasticsearch_jvm_memory_pools_survivor_used_in_bytes` | GaugeValue | JVM幸存区使用的内存量（字节） | +| `elasticsearch_jvm_memory_pools_survivor_max_in_bytes` | CounterValue | JVM幸存区最大内存量（字节） | +| `elasticsearch_jvm_memory_pools_survivor_peak_used_in_bytes` | CounterValue | JVM幸存区峰值使用的内存量（字节 | + +#### `export_indices = true` + +| 名称 | 类型 | 描述 | +|----------------------------------------------------------------------------|--------------|-----------------------------| +| `elasticsearch_indices_stats_total_docs_count` | GaugeValue | 文档总数 | +| `elasticsearch_indices_stats_total_docs_deleted` | GaugeValue | 已删除的文档总数 | +| `elasticsearch_indices_stats_total_store_size_in_bytes` | GaugeValue | 当前所有节点上所有分片存储的索引数据的总大小（字节） | +| `elasticsearch_indices_stats_total_throttle_time_seconds` | GaugeValue | 索引被节流的总时间（秒） | +| `elasticsearch_indices_stats_total_segments_count` | GaugeValue | 当前所有节点上所有分片的段数量 | +| `elasticsearch_indices_stats_total_segments_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的段占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_terms_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的词项占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_stored_fields_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的存储字段占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_term_vectors_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的词向量占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_norms_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的规范化值占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_points_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的点数据占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_doc_values_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的文档值占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_index_writer_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的索引编写器占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_version_map_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的版本映射占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_fixed_bit_set_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的固定位集占用内存大小（字节） | +| `elasticsearch_indices_stats_total_segments_max_unsafe_auto_id_timestamp` | GaugeValue | 当前所有节点上所有分片的最大不安全自动ID时间戳 | +| `elasticsearch_indices_stats_total_translog_earliest_last_modified_age` | GaugeValue | 当前所有节点上所有分片的事务日志中最早上次修改的年龄 | +| `elasticsearch_indices_stats_total_translog_operations` | GaugeValue | 当前所有节点上所有分片的事务日志操作数量 | +| `elasticsearch_indices_stats_total_translog_size_in_bytes` | GaugeValue | 当前所有节点上所有分片的事务日志大小（字节） | +| `elasticsearch_indices_stats_total_translog_uncommitted_operations` | GaugeValue | 当前所有节点上所有分片的未提交的事务日志操作数量 | +| `elasticsearch_indices_stats_total_translog_uncommitted_size_in_bytes` | GaugeValue | 当前所有节点上所有分片的未提交事务日志大小（字节） | +| `elasticsearch_indices_stats_total_completion_size_in_bytes` | GaugeValue | 当前所有节点上所有分片的自动完成数据大小（字节） | +| `elasticsearch_indices_stats_total_search_query_time_seconds` | CounterValue | 搜索查询总时间（秒） | +| `elasticsearch_indices_stats_total_search_query_current` | GaugeValue | 当前活跃的搜索查询数量 | +| `elasticsearch_indices_stats_total_search_open_contexts` | CounterValue | 打开的搜索上下文总数 | +| `elasticsearch_indices_stats_total_search_query_total` | CounterValue | 搜索查询总数 | +| `elasticsearch_indices_stats_total_search_fetch_time_seconds` | CounterValue | 搜索抓取总时间（秒） | +| `elasticsearch_indices_stats_total_search_fetch` | CounterValue | 搜索抓取总次数 | +| `elasticsearch_indices_stats_total_search_fetch_current` | CounterValue | 当前搜索抓取次数 | +| `elasticsearch_indices_stats_total_search_scroll_time_seconds` | CounterValue | 搜索滚动总时间（秒） | +| `elasticsearch_indices_stats_total_search_scroll_current` | GaugeValue | 当前搜索滚动次数 | +| `elasticsearch_indices_stats_total_search_scroll` | CounterValue | 搜索滚动总次数 | +| `elasticsearch_indices_stats_total_search_suggest_time_seconds` | CounterValue | 搜索建议总时间（秒） | +| `elasticsearch_indices_stats_total_search_suggest_total` | CounterValue | 搜索建议总次数 | +| `elasticsearch_indices_stats_total_search_suggest_current` | CounterValue | 当前搜索建议次数 | +| `elasticsearch_indices_stats_total_indexing_index_time_seconds` | CounterValue | 索引索引总时间（秒） | +| `elasticsearch_indices_stats_total_index_current` | GaugeValue | 当前正在索引的文档数 | +| `elasticsearch_indices_stats_total_index_failed` | GaugeValue | 索引失败的文档数 | +| `elasticsearch_indices_stats_total_delete_current` | GaugeValue | 当前正在处理的删除操作数 | +| `elasticsearch_indices_stats_total_indexing_index` | CounterValue | 索引索引操作总次数 | +| `elasticsearch_indices_stats_total_indexing_delete_time_seconds` | CounterValue | 索引删除操作总时间（秒） | +| `elasticsearch_indices_stats_total_indexing_delete` | CounterValue | 索引删除操作总次数 | +| `elasticsearch_indices_stats_total_indexing_noop_update` | CounterValue | 无操作更新总次数 | +| `elasticsearch_indices_stats_total_indexing_throttle_time_seconds` | CounterValue | 索引节流总时间（秒） | +| `elasticsearch_indices_stats_total_get_time_seconds` | CounterValue | 获取操作总时间（秒） | +| `elasticsearch_indices_stats_total_get_exists_total` | CounterValue | 存在检查操作总次数 | +| `elasticsearch_indices_stats_total_get_exists_time_seconds` | CounterValue | 存在检查操作总时间（秒） | +| `elasticsearch_indices_stats_total_get_total` | CounterValue | 获取操作总次数 | +| `elasticsearch_indices_stats_total_get_missing_total` | CounterValue | 缺失检查操作总次数 | +| `elasticsearch_indices_stats_total_get_missing_time_seconds` | CounterValue | 缺失检查操作总时间（秒） | +| `elasticsearch_indices_stats_total_get_current` | CounterValue | 当前获取操作次数 | +| `elasticsearch_indices_stats_total_merges_time_seconds` | CounterValue | 合并操作总时间（秒） | +| `elasticsearch_indices_stats_total_merges_total` | CounterValue | 合并操作总次数 | +| `elasticsearch_indices_stats_total_merges_total_docs` | CounterValue | 合并操作处理的文档总数 | +| `elasticsearch_indices_stats_total_merges_total_size_in_bytes` | CounterValue | 合并操作处理的数据总大小（字节） | +| `elasticsearch_indices_stats_total_merges_current` | CounterValue | 当前合并操作数 | +| `elasticsearch_indices_stats_total_merges_current_docs` | CounterValue | 当前合并操作处理的文档数 | +| `elasticsearch_indices_stats_total_merges_current_size_in_bytes` | CounterValue | 当前合并操作处理的数据大小（字节） | +| `elasticsearch_indices_stats_total_merges_total_throttle_time_seconds` | CounterValue | 合并操作I/O节流总时间（秒） | +| `elasticsearch_indices_stats_total_merges_total_stopped_time_seconds` | CounterValue | 允许较小合并完成的总大型合并停止时间（秒） | +| `elasticsearch_indices_stats_total_merges_total_auto_throttle_bytes` | CounterValue | 合并期间自动节流的总字节数 | +| `elasticsearch_indices_stats_total_refresh_external_total_time_seconds` | CounterValue | 外部刷新总时间（秒） | +| `elasticsearch_indices_stats_total_refresh_external_total` | CounterValue | 外部刷新总次数 | +| `elasticsearch_indices_stats_total_refresh_total_time_seconds` | CounterValue | 刷新操作总时间（秒） | +| `elasticsearch_indices_stats_total_refresh_total` | CounterValue | 刷新操作总次数 | +| `elasticsearch_indices_stats_total_refresh_listeners` | CounterValue | 刷新监听器总数 | +| `elasticsearch_indices_stats_total_recovery_current_as_source` | CounterValue | 作为源的当前恢复操作数 | +| `elasticsearch_indices_stats_total_recovery_current_as_target` | CounterValue | 作为目标的当前恢复操作数 | +| `elasticsearch_indices_stats_total_recovery_throttle_time_seconds` | CounterValue | 恢复操作节流总时间（秒） | +| `elasticsearch_indices_stats_total_flush_time_seconds_total` | CounterValue | 刷新操作总时间（秒） | +| `elasticsearch_indices_stats_total_flush_total` | CounterValue | 刷新操作总次数 | +| `elasticsearch_indices_stats_total_flush_periodic` | CounterValue | 周期性刷新总次数 | +| `elasticsearch_indices_stats_total_warmer_time_seconds_total` | CounterValue | 预热操作总时间（秒） | +| `elasticsearch_indices_stats_total_warmer_total` | CounterValue | 预热操作总次数 | +| `elasticsearch_indices_stats_total_query_cache_memory_in_bytes` | CounterValue | 查询缓存总内存（字节） | +| `elasticsearch_indices_stats_total_query_cache_size` | GaugeValue | 查询缓存总大小 | +| `elasticsearch_indices_stats_total_query_cache_total_count` | CounterValue | 查询缓存操作总次数 | +| `elasticsearch_indices_stats_total_query_cache_hit_count` | CounterValue | 查询缓存命中总次数 | +| `elasticsearch_indices_stats_total_query_cache_miss_count` | CounterValue | 查询缓存未命中总次数 | +| `elasticsearch_indices_stats_total_query_cache_cache_count` | CounterValue | 查询缓存缓存总次数 | +| `elasticsearch_indices_stats_total_query_cache_evictions` | CounterValue | 查询缓存逐出总次数 | +| `elasticsearch_indices_stats_total_request_cache_memory_in_bytes` | CounterValue | 请求缓存总内存（字节） | +| `elasticsearch_indices_stats_total_request_cache_hit_count` | CounterValue | 请求缓存命中总次数 | +| `elasticsearch_indices_stats_total_request_cache_miss_count` | CounterValue | 请求缓存未命中总次数 | +| `elasticsearch_indices_stats_total_request_cache_evictions` | CounterValue | 请求缓存逐出总次数 | +| `elasticsearch_indices_stats_total_fielddata_memory_in_bytes` | CounterValue | 字段数据总内存（字节） | +| `elasticsearch_indices_stats_total_fielddata_evictions` | CounterValue | 字段数据逐出总次数 | +| `elasticsearch_indices_stats_total_seq_no_global_checkpoint` | CounterValue | 全局检查点 | +| `elasticsearch_indices_stats_total_seq_no_local_checkpoint` | CounterValue | 本地检查点 | +| `elasticsearch_indices_stats_total_seq_no_max_seq_no` | CounterValue | 最大序列号 | + +| 名称 | 类型 | 描述 | +|---------------------------------------------------------------------------------|--------------|-----------------------------| +| `elasticsearch_indices_stats_primaries_docs_count` | GaugeValue | 文档总数 | +| `elasticsearch_indices_stats_primaries_docs_deleted` | GaugeValue | 已删除的文档总数 | +| `elasticsearch_indices_stats_primaries_store_size_in_bytes` | GaugeValue | 当前所有节点上所有分片存储的索引数据的总大小（字节） | +| `elasticsearch_indices_stats_primaries_throttle_time_seconds` | GaugeValue | 索引被节流的总时间（秒） | +| `elasticsearch_indices_stats_primaries_segments_count` | GaugeValue | 当前所有节点上所有分片的段数量 | +| `elasticsearch_indices_stats_primaries_segments_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的段占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_terms_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的词项占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_stored_fields_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的存储字段占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_term_vectors_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的词向量占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_norms_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的规范化值占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_points_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的点数据占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_doc_values_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的文档值占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_index_writer_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的索引编写器占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_version_map_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的版本映射占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_fixed_bit_set_memory_in_bytes` | GaugeValue | 当前所有节点上所有分片的固定位集占用内存大小（字节） | +| `elasticsearch_indices_stats_primaries_segments_max_unsafe_auto_id_timestamp` | GaugeValue | 当前所有节点上所有分片的最大不安全自动ID时间戳 | +| `elasticsearch_indices_stats_primaries_translog_earliest_last_modified_age` | GaugeValue | 当前所有节点上所有分片的事务日志中最早上次修改的年龄 | +| `elasticsearch_indices_stats_primaries_translog_operations` | GaugeValue | 当前所有节点上所有分片的事务日志操作数量 | +| `elasticsearch_indices_stats_primaries_translog_size_in_bytes` | GaugeValue | 当前所有节点上所有分片的事务日志大小（字节） | +| `elasticsearch_indices_stats_primaries_translog_uncommitted_operations` | GaugeValue | 当前所有节点上所有分片的未提交的事务日志操作数量 | +| `elasticsearch_indices_stats_primaries_translog_uncommitted_size_in_bytes` | GaugeValue | 当前所有节点上所有分片的未提交事务日志大小（字节） | +| `elasticsearch_indices_stats_primaries_completion_size_in_bytes` | GaugeValue | 当前所有节点上所有分片的自动完成数据大小（字节） | +| `elasticsearch_indices_stats_primaries_search_query_time_seconds` | CounterValue | 搜索查询总时间（秒） | +| `elasticsearch_indices_stats_primaries_search_query_current` | GaugeValue | 当前活跃的搜索查询数量 | +| `elasticsearch_indices_stats_primaries_search_open_contexts` | CounterValue | 打开的搜索上下文总数 | +| `elasticsearch_indices_stats_primaries_search_query_total` | CounterValue | 搜索查询总数 | +| `elasticsearch_indices_stats_primaries_search_fetch_time_seconds` | CounterValue | 搜索抓取总时间（秒） | +| `elasticsearch_indices_stats_primaries_search_fetch` | CounterValue | 搜索抓取总次数 | +| `elasticsearch_indices_stats_primaries_search_fetch_current` | CounterValue | 当前搜索抓取次数 | +| `elasticsearch_indices_stats_primaries_search_scroll_time_seconds` | CounterValue | 搜索滚动总时间（秒） | +| `elasticsearch_indices_stats_primaries_search_scroll_current` | GaugeValue | 当前搜索滚动次数 | +| `elasticsearch_indices_stats_primaries_search_scroll` | CounterValue | 搜索滚动总次数 | +| `elasticsearch_indices_stats_primaries_search_suggest_time_seconds` | CounterValue | 搜索建议总时间（秒） | +| `elasticsearch_indices_stats_primaries_search_suggest_total` | CounterValue | 搜索建议总次数 | +| `elasticsearch_indices_stats_primaries_search_suggest_current` | CounterValue | 当前搜索建议次数 | +| `elasticsearch_indices_stats_primaries_indexing_index_time_seconds` | CounterValue | 索引索引总时间（秒） | +| `elasticsearch_indices_stats_primaries_index_current` | GaugeValue | 当前正在索引的文档数 | +| `elasticsearch_indices_stats_primaries_index_failed` | GaugeValue | 索引失败的文档数 | +| `elasticsearch_indices_stats_primaries_delete_current` | GaugeValue | 当前正在处理的删除操作数 | +| `elasticsearch_indices_stats_primaries_indexing_index` | CounterValue | 索引索引操作总次数 | +| `elasticsearch_indices_stats_primaries_indexing_delete_time_seconds` | CounterValue | 索引删除操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_indexing_delete` | CounterValue | 索引删除操作总次数 | +| `elasticsearch_indices_stats_primaries_indexing_noop_update` | CounterValue | 无操作更新总次数 | +| `elasticsearch_indices_stats_primaries_indexing_throttle_time_seconds` | CounterValue | 索引节流总时间（秒） | +| `elasticsearch_indices_stats_primaries_get_time_seconds` | CounterValue | 获取操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_get_exists_total` | CounterValue | 存在检查操作总次数 | +| `elasticsearch_indices_stats_primaries_get_exists_time_seconds` | CounterValue | 存在检查操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_get_total` | CounterValue | 获取操作总次数 | +| `elasticsearch_indices_stats_primaries_get_missing_total` | CounterValue | 缺失检查操作总次数 | +| `elasticsearch_indices_stats_primaries_get_missing_time_seconds` | CounterValue | 缺失检查操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_get_current` | CounterValue | 当前获取操作次数 | +| `elasticsearch_indices_stats_primaries_merges_time_seconds` | CounterValue | 合并操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_merges_total` | CounterValue | 合并操作总次数 | +| `elasticsearch_indices_stats_primaries_merges_primaries_docs` | CounterValue | 合并操作处理的文档总数 | +| `elasticsearch_indices_stats_primaries_merges_primaries_size_in_bytes` | CounterValue | 合并操作处理的数据总大小（字节） | +| `elasticsearch_indices_stats_primaries_merges_current` | CounterValue | 当前合并操作数 | +| `elasticsearch_indices_stats_primaries_merges_current_docs` | CounterValue | 当前合并操作处理的文档数 | +| `elasticsearch_indices_stats_primaries_merges_current_size_in_bytes` | CounterValue | 当前合并操作处理的数据大小（字节） | +| `elasticsearch_indices_stats_primaries_merges_primaries_throttle_time_seconds` | CounterValue | 合并操作I/O节流总时间（秒） | +| `elasticsearch_indices_stats_primaries_merges_primaries_stopped_time_seconds` | CounterValue | 允许较小合并完成的总大型合并停止时间（秒） | +| `elasticsearch_indices_stats_primaries_merges_primaries_auto_throttle_bytes` | CounterValue | 合并期间自动节流的总字节数 | +| `elasticsearch_indices_stats_primaries_refresh_external_primaries_time_seconds` | CounterValue | 外部刷新总时间（秒） | +| `elasticsearch_indices_stats_primaries_refresh_external_total` | CounterValue | 外部刷新总次数 | +| `elasticsearch_indices_stats_primaries_refresh_primaries_time_seconds` | CounterValue | 刷新操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_refresh_total` | CounterValue | 刷新操作总次数 | +| `elasticsearch_indices_stats_primaries_refresh_listeners` | CounterValue | 刷新监听器总数 | +| `elasticsearch_indices_stats_primaries_recovery_current_as_source` | CounterValue | 作为源的当前恢复操作数 | +| `elasticsearch_indices_stats_primaries_recovery_current_as_target` | CounterValue | 作为目标的当前恢复操作数 | +| `elasticsearch_indices_stats_primaries_recovery_throttle_time_seconds` | CounterValue | 恢复操作节流总时间（秒） | +| `elasticsearch_indices_stats_primaries_flush_time_seconds_total` | CounterValue | 刷新操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_flush_total` | CounterValue | 刷新操作总次数 | +| `elasticsearch_indices_stats_primaries_flush_periodic` | CounterValue | 周期性刷新总次数 | +| `elasticsearch_indices_stats_primaries_warmer_time_seconds_total` | CounterValue | 预热操作总时间（秒） | +| `elasticsearch_indices_stats_primaries_warmer_total` | CounterValue | 预热操作总次数 | +| `elasticsearch_indices_stats_primaries_query_cache_memory_in_bytes` | CounterValue | 查询缓存总内存（字节） | +| `elasticsearch_indices_stats_primaries_query_cache_size` | GaugeValue | 查询缓存总大小 | +| `elasticsearch_indices_stats_primaries_query_cache_primaries_count` | CounterValue | 查询缓存操作总次数 | +| `elasticsearch_indices_stats_primaries_query_cache_hit_count` | CounterValue | 查询缓存命中总次数 | +| `elasticsearch_indices_stats_primaries_query_cache_miss_count` | CounterValue | 查询缓存未命中总次数 | +| `elasticsearch_indices_stats_primaries_query_cache_cache_count` | CounterValue | 查询缓存缓存总次数 | +| `elasticsearch_indices_stats_primaries_query_cache_evictions` | CounterValue | 查询缓存逐出总次数 | +| `elasticsearch_indices_stats_primaries_request_cache_memory_in_bytes` | CounterValue | 请求缓存总内存（字节） | +| `elasticsearch_indices_stats_primaries_request_cache_hit_count` | CounterValue | 请求缓存命中总次数 | +| `elasticsearch_indices_stats_primaries_request_cache_miss_count` | CounterValue | 请求缓存未命中总次数 | +| `elasticsearch_indices_stats_primaries_request_cache_evictions` | CounterValue | 请求缓存逐出总次数 | +| `elasticsearch_indices_stats_primaries_fielddata_memory_in_bytes` | CounterValue | 字段数据总内存（字节） | +| `elasticsearch_indices_stats_primaries_fielddata_evictions` | CounterValue | 字段数据逐出总次数 | +| `elasticsearch_indices_stats_primaries_seq_no_global_checkpoint` | CounterValue | 全局检查点 | +| `elasticsearch_indices_stats_primaries_seq_no_local_checkpoint` | CounterValue | 本地检查点 | +| `elasticsearch_indices_stats_primaries_seq_no_max_seq_no` | CounterValue | 最大序列号 | + +#### `export_indices_settings = true` + +| 名称 | 类型 | 帮助 | +|-----------------------------------------------------------|-------|-------------------------------------------------------| +| elasticsearch_indices_settings_creation_timestamp_seconds | gauge | 索引创建时间的时间戳，单位为秒 | +| elasticsearch_indices_settings_stats_read_only_indices | gauge | 设置为read_only_allow_delete=true的索引数量 | +| elasticsearch_indices_settings_total_fields | gauge | 索引设置中index.mapping.total_fields.limit的值（索引中允许的映射字段总数） | +| elasticsearch_indices_settings_replicas | gauge | 索引设置中index.replicas的值 | + +#### `export_indices_mappings = true` + +| 名称 | 类型 | 帮助 | +|----------------------------------------------------------------|---------|------------------------------| +| elasticsearch_indices_mappings_stats_fields | gauge | 索引当前映射的字段数 | +| elasticsearch_indices_mappings_stats_json_parse_failures_total | counter | 解析JSON时的错误数 | +| elasticsearch_indices_mappings_stats_scrapes_total | counter | 当前Elasticsearch索引映射抓取的总次数 | +| elasticsearch_indices_mappings_stats_up | gauge | 上一次抓取Elasticsearch索引映射端点是否成功 | + +#### `export_slm = true` + +| 名称 | 类型 | 帮助 | +|----------------------------------------------------------|---------|----------------------| +| elasticsearch_slm_stats_up | gauge | SLM收集器的上行指标 | +| elasticsearch_slm_stats_total_scrapes | counter | SLM收集器的抓取次数 | +| elasticsearch_slm_stats_json_parse_failures | counter | SLM收集器的JSON解析失败次数 | +| elasticsearch_slm_stats_retention_runs_total | counter | 保留运行总次数 | +| elasticsearch_slm_stats_retention_failed_total | counter | 保留运行失败总次数 | +| elasticsearch_slm_stats_retention_timed_out_total | counter | 保留运行超时总次数 | +| elasticsearch_slm_stats_retention_deletion_time_seconds | gauge | 保留运行删除时间 | +| elasticsearch_slm_stats_total_snapshots_taken_total | counter | 总共拍摄的快照数 | +| elasticsearch_slm_stats_total_snapshots_failed_total | counter | 快照失败总次数 | +| elasticsearch_slm_stats_total_snapshots_deleted_total | counter | 快照删除总次数 | +| elasticsearch_slm_stats_snapshots_taken_total | counter | 按策略拍摄的快照数 | +| elasticsearch_slm_stats_snapshots_failed_total | counter | 按策略失败的快照数 | +| elasticsearch_slm_stats_snapshots_deleted_total | counter | 按策略删除的快照数 | +| elasticsearch_slm_stats_snapshot_deletion_failures_total | counter | 按策略快照删除失败次数 | +| elasticsearch_slm_stats_operation_mode | gauge | SLM操作模式（运行中，停止中，已停止） | diff --git a/inputs/elasticsearch/README_en.md b/inputs/elasticsearch/README_en.md new file mode 100644 index 00000000..ff374c42 --- /dev/null +++ b/inputs/elasticsearch/README_en.md @@ -0,0 +1,479 @@ +# Elasticsearch Exporter + +#### Elasticsearch 7.x security privileges + +Username and password can be passed either directly in the URI or through the `ES_USERNAME` and `ES_PASSWORD` environment variables. +Specifying those two environment variables will override authentication passed in the URI (if any). + +ES 7.x supports RBACs. The following security privileges are required for the `elasticsearch` plugin. + +| Setting | Privilege Required | Description | +|:------------------------|:-------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------| +| export_cluster_settings | `cluster` `monitor` | | +| exporter defaults | `cluster` `monitor` | All cluster read-only operations, like cluster health and state, hot threads, node info, node and cluster stats, and pending cluster tasks. | +| export_indices | `indices` `monitor` (per index or `*`) | All actions that are required for monitoring (recovery, segments info, index stats and status) | +| export_indices_settings | `indices` `monitor` (per index or `*`) | | +| export_indices_mappings | `indices` `view_index_metadata` (per index or `*`) | | +| export_shards | not sure if `indices` or `cluster` `monitor` or both | | +| export_snapshots | `cluster:admin/snapshot/status` and `cluster:admin/repository/get` | [ES Forum Post](https://discuss.elastic.co/t/permissions-for-backup-user-with-x-pack/88057) | +| export_slm | `read_slm` | | +| export_data_stream | `monitor` or `manage` (per index or `*`) | | + +### Differences between the old version of `elastisearch` plugin and the new one + +- `elasticsearch_cluster_health_active_shards_percent_as_number` has been changed to `elasticsearch_cluster_health_active_shards_percent`. +- `elasticsearch_cluster_health_status` and `elasticsearch_cluster_health_status_code` have been merged into `elasticsearch_cluster_health_status`, with values being `green=1`, `yellow=2`, `red=3`. +- `elasticsearch_process_cpu_total_in_millis` has been changed to `elasticsearch_process_cpu_seconds_total`, with the unit being seconds. +- `elasticsearch_jvm_uptime_in_millis` has been changed to `elasticsearch_jvm_uptime_seconds`, with the unit being seconds. Similarly, all metrics ending with `*_in_millis` have been changed to `*_seconds`. + +### Metrics + +#### `cluster_health = true` + +| Name | Type | Description | +|-----------------------------------------------------------------|------------|--------------------------------------------------------------------------------------------------| +| `elasticsearch_cluster_health_active_primary_shards` | GaugeValue | The number of primary shards in your cluster. This is an aggregate total across all indices. | +| `elasticsearch_cluster_health_active_shards` | GaugeValue | Aggregate total of all shards across all indices, which includes replica shards. | +| `elasticsearch_cluster_health_active_shards_percent` | GaugeValue | Percentage of active shards in the cluster. | +| `elasticsearch_cluster_health_delayed_unassigned_shards` | GaugeValue | Shards delayed to reduce reallocation overhead. | +| `elasticsearch_cluster_health_initializing_shards` | GaugeValue | Count of shards that are being freshly created. | +| `elasticsearch_cluster_health_number_of_data_nodes` | GaugeValue | Number of data nodes in the cluster. | +| `elasticsearch_cluster_health_number_of_in_flight_fetch` | GaugeValue | The number of ongoing shard info requests. | +| `elasticsearch_cluster_health_task_max_waiting_in_queue_millis` | GaugeValue | Tasks max time waiting in queue. | +| `elasticsearch_cluster_health_number_of_nodes` | GaugeValue | Number of nodes in the cluster. | +| `elasticsearch_cluster_health_number_of_pending_tasks` | GaugeValue | Cluster level changes which have not yet been executed. | +| `elasticsearch_cluster_health_relocating_shards` | GaugeValue | The number of shards that are currently moving from one node to another node. | +| `elasticsearch_cluster_health_unassigned_shards` | GaugeValue | The number of shards that exist in the cluster state, but cannot be found in the cluster itself. | + +#### `cluster_health = true` and `cluster_health_level = "indices"` + +| Name | Type | Description | +|--------------------------------------------------------------|------------|--------------------------------------------------------------------------------------------------| +| `elasticsearch_cluster_health_indices_active_primary_shards` | GaugeValue | The number of primary shards in your cluster. This is an aggregate total across all indices. | +| `elasticsearch_cluster_health_indices_active_shards` | GaugeValue | Aggregate total of all shards across all indices, which includes replica shards. | +| `elasticsearch_cluster_health_indices_initializing_shards` | GaugeValue | Count of shards that are being freshly created. | +| `elasticsearch_cluster_health_indices_number_of_replicas` | GaugeValue | Number of replicas in the cluster. | +| `elasticsearch_cluster_health_indices_number_of_shards` | GaugeValue | Number of shards in the cluster. | +| `elasticsearch_cluster_health_indices_relocating_shards` | GaugeValue | The number of shards that are currently moving from one node to another node. | +| `elasticsearch_cluster_health_indices_unassigned_shards` | GaugeValue | The number of shards that exist in the cluster state, but cannot be found in the cluster itself. | + +#### `export_cluster_settings = true` + +| Name | Type | Description | +|------------------------------------------------------------------------|------------|-----------------------------------------------------------------| +| `elasticsearch_clustersettings_stats_shard_allocation_enabled` | GaugeValue | Current mode of cluster wide shard routing allocation settings. | +| `elasticsearch_clustersettings_stats_max_shards_per_node` | GaugeValue | Current maximum number of shards per node setting. | +| `elasticsearch_clustersettings_allocation_threshold_enabled` | GaugeValue | Is disk allocation decider enabled. | +| `elasticsearch_clustersettings_allocation_watermark_flood_stage_ratio` | GaugeValue | Flood stage watermark as a ratio. | +| `elasticsearch_clustersettings_allocation_watermark_high_ratio` | GaugeValue | High watermark for disk usage as a ratio. | +| `elasticsearch_clustersettings_allocation_watermark_low_ratio` | GaugeValue | Low watermark for disk usage as a ratio. | +| `elasticsearch_clustersettings_allocation_watermark_flood_stage_bytes` | GaugeValue | Flood stage watermark in bytes. | +| `elasticsearch_clustersettings_allocation_watermark_high_bytes` | GaugeValue | High watermark for disk usage in bytes. | +| `elasticsearch_clustersettings_allocation_watermark_low_bytes` | GaugeValue | Low watermark for disk usage in bytes. | + +#### `cluster_stats = true` + +| Name | Type | Description | +|------------------------------------------------------------------------------------------|---------------|--------------------------------------------------| +| `elasticsearch_clusterstats_indices_count` | CounterValue | Completion in bytes | +| `elasticsearch_clusterstats_indices_completion_size_in_bytes` | CounterValue | Completion in bytes | +| `elasticsearch_clusterstats_indices_docs_count` | GaugeValue | Count of documents on this cluster | +| `elasticsearch_clusterstats_indices_docs_deleted` | GaugeValue | Count of deleted documents on this cluster | +| `elasticsearch_clusterstats_indices_fielddata_evictions` | CounterValue | Evictions from field data | +| `elasticsearch_clusterstats_indices_fielddata_memory_size_in_bytes` | GaugeValue | Field data cache memory usage in bytes | +| `elasticsearch_clusterstats_indices_query_cache_cache_count` | CounterValue | Query cache cache count | +| `elasticsearch_clusterstats_indices_query_cache_cache_size` | GaugeValue | Query cache cache size | +| `elasticsearch_clusterstats_indices_query_cache_evictions` | CounterValue | Evictions from query cache | +| `elasticsearch_clusterstats_indices_query_cache_hit_count` | CounterValue | Query cache count | +| `elasticsearch_clusterstats_indices_query_cache_memory_size_in_bytes` | GaugeValue | Query cache memory usage in bytes | +| `elasticsearch_clusterstats_indices_query_cache_miss_count` | CounterValue | Query miss count | +| `elasticsearch_clusterstats_indices_query_cache_total_count` | CounterValue | Query cache total count | +| `elasticsearch_clusterstats_indices_segments_count` | GaugeValue | Count of index segments on this cluster | +| `elasticsearch_clusterstats_indices_segments_doc_values_memory_in_bytes` | GaugeValue | Count of doc values memory | +| `elasticsearch_clusterstats_indices_segments_fixed_bit_set_memory_in_bytes` | GaugeValue | Count of fixed bit set | +| `elasticsearch_clusterstats_indices_segments_index_writer_memory_in_bytes` | GaugeValue | Count of memory for index writer on this cluster | +| `elasticsearch_clusterstats_indices_segments_max_unsafe_auto_id_timestamp` | GaugeValue | Count of memory for index writer on this cluster | +| `elasticsearch_clusterstats_indices_segments_memory_in_bytes` | GaugeValue | Current memory size of segments in bytes | +| `elasticsearch_clusterstats_indices_segments_norms_memory_in_bytes` | GaugeValue | Count of memory used by norms | +| `elasticsearch_clusterstats_indices_segments_points_memory_in_bytes` | GaugeValue | Point values memory usage in bytes | +| `elasticsearch_clusterstats_indices_segments_stored_fields_memory_in_bytes` | GaugeValue | Count of stored fields memory | +| `elasticsearch_clusterstats_indices_segments_term_vectors_memory_in_bytes` | GaugeValue | Term vectors memory usage in bytes | +| `elasticsearch_clusterstats_indices_segments_terms_memory_in_bytes` | GaugeValue | Count of terms in memory for this cluster | +| `elasticsearch_clusterstats_indices_segments_version_map_memory_in_bytes` | GaugeValue | Version map memory usage in bytes | +| `elasticsearch_clusterstats_indices_shards_total` | GaugeValue | Total number of shards in the cluster | +| `elasticsearch_clusterstats_indices_shards_replication` | GaugeValue | Number of shards replication | +| `elasticsearch_clusterstats_indices_shards_primaries` | GaugeValue | Number of primary shards in the cluster | +| `elasticsearch_clusterstats_indices_shards_index_primaries_avg` | GaugeValue | Average number of primary shards per index | +| `elasticsearch_clusterstats_indices_shards_index_primaries_max` | GaugeValue | Max number of primary shards per index | +| `elasticsearch_clusterstats_indices_shards_index_primaries_min` | GaugeValue | Min number of primary shards per index | +| `elasticsearch_clusterstats_indices_shards_index_replication_avg` | GaugeValue | Average number of replication shards per index | +| `elasticsearch_clusterstats_indices_shards_index_replication_max` | GaugeValue | Max number of replication shards per index | +| `elasticsearch_clusterstats_indices_shards_index_replication_min` | GaugeValue | Min number of replication shards per index | +| `elasticsearch_clusterstats_indices_shards_index_shards_avg` | GaugeValue | Average number of shards per index | +| `elasticsearch_clusterstats_indices_shards_index_shards_max` | GaugeValue | Max number of shards per index | +| `elasticsearch_clusterstats_indices_shards_index_shards_min` | GaugeValue | Min number of shards per index | +| `elasticsearch_clusterstats_indices_store_size_in_bytes` | GaugeValue | Current size of the store in bytes | +| `elasticsearch_clusterstats_indices_total_data_set_size_in_bytes` | GaugeValue | Total data set size in bytes | +| `elasticsearch_clusterstats_indices_reserved_in_bytes` | GaugeValue | Reserved size in bytes | +| `elasticsearch_clusterstats_nodes_count_coordinating_only` | GaugeValue | Count of coordinating only nodes | +| `elasticsearch_clusterstats_nodes_count_data` | GaugeValue | Count of data nodes | +| `elasticsearch_clusterstats_nodes_count_ingest` | GaugeValue | Count of ingest nodes | +| `elasticsearch_clusterstats_nodes_count_master` | GaugeValue | Count of master nodes | +| `elasticsearch_clusterstats_nodes_count_total` | GaugeValue | Total count of nodes in the cluster | +| `elasticsearch_clusterstats_nodes_fs_available_in_bytes` | GaugeValue | Available disk space in bytes | +| `elasticsearch_clusterstats_nodes_fs_free_in_bytes` | GaugeValue | Free disk space in bytes | +| `elasticsearch_clusterstats_nodes_fs_total_in_bytes` | GaugeValue | Total disk space in bytes | +| `elasticsearch_clusterstats_nodes_jvm_max_uptime_in_millis` | GaugeValue | Max uptime in milliseconds | +| `elasticsearch_clusterstats_nodes_jvm_mem_heap_max_in_bytes` | GaugeValue | Max heap memory in bytes | +| `elasticsearch_clusterstats_nodes_jvm_mem_heap_used_in_bytes` | GaugeValue | Used heap memory in bytes | +| `elasticsearch_clusterstats_nodes_jvm_threads` | GaugeValue | Number of threads | +| `elasticsearch_clusterstats_nodes_network_types_http_types_security4` | GaugeValue | HTTP security4 network types | +| `elasticsearch_clusterstats_nodes_network_types_transport_types_security4` | GaugeValue | Transport security4 network types | +| `elasticsearch_clusterstats_nodes_os_allocated_processors` | GaugeValue | Allocated processors | +| `elasticsearch_clusterstats_nodes_os_available_processors` | GaugeValue | Available processors | +| `elasticsearch_clusterstats_nodes_os_mem_free_in_bytes` | GaugeValue | Free memory in bytes | +| `elasticsearch_clusterstats_nodes_os_mem_free_percent` | GaugeValue | Free memory in percent | +| `elasticsearch_clusterstats_nodes_os_mem_total_in_bytes` | GaugeValue | Total memory in bytes | +| `elasticsearch_clusterstats_nodes_os_mem_used_in_bytes` | GaugeValue | Used memory in bytes | +| `elasticsearch_clusterstats_nodes_os_mem_used_percent` | GaugeValue | Used memory in percent | +| `elasticsearch_clusterstats_nodes_process_cpu_percent` | GaugeValue | Process CPU in percent | +| `elasticsearch_clusterstats_nodes_process_open_file_descriptors_avg` | GaugeValue | Average number of open file descriptors | +| `elasticsearch_clusterstats_nodes_process_open_file_descriptors_max` | GaugeValue | Max number of open file descriptors | +| `elasticsearch_clusterstats_nodes_process_open_file_descriptors_min` | GaugeValue | Min number of open file descriptors | + +#### `export_data_stream = true` + +| Name | Type | Description | +|-------------------------------------------------------|--------------|--------------------------------------------------| +| `elasticsearch_data_stream_backing_indices_total` | CounterValue | Number of backing indices | +| `elasticsearch_data_stream_store_size_bytes` | CounterValue | Store size of data stream | +| `elasticsearch_data_stream_stats_up` | gauge | Up metric for Data Stream collection | +| `elasticsearch_data_stream_stats_total_scrapes` | counter | Total scrapes for Data Stream stats | +| `elasticsearch_data_stream_stats_json_parse_failures` | counter | Number of parsing failures for Data Stream stats | + +#### `export_indices = true` + +| Name | Type | Description | +|----------------------------------------------------------------------------|--------------|----------------------------------------------------------------------------------------------| +| `elasticsearch_indices_stats_total_docs_count` | GaugeValue | Total count of documents | +| `elasticsearch_indices_stats_total_docs_deleted` | GaugeValue | Total count of deleted documents | +| `elasticsearch_indices_stats_total_store_size_in_bytes` | GaugeValue | Current total size of stored index data in bytes with all shards on all nodes | +| `elasticsearch_indices_stats_total_throttle_time_seconds` | GaugeValue | Total time the index has been throttled in seconds | +| `elasticsearch_indices_stats_total_segments_count` | GaugeValue | Current number of segments with all shards on all nodes | +| `elasticsearch_indices_stats_total_segments_memory_in_bytes` | GaugeValue | Current size of segments with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_terms_memory_in_bytes` | GaugeValue | Current number of terms with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_stored_fields_memory_in_bytes` | GaugeValue | Current size of fields with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_term_vectors_memory_in_bytes` | GaugeValue | Current size of term vectors with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_norms_memory_in_bytes` | GaugeValue | Current size of norms with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_points_memory_in_bytes` | GaugeValue | Current size of points with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_doc_values_memory_in_bytes` | GaugeValue | Current size of doc values with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_index_writer_memory_in_bytes` | GaugeValue | Current size of index writer with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_version_map_memory_in_bytes` | GaugeValue | Current size of version map with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_fixed_bit_set_memory_in_bytes` | GaugeValue | Current size of fixed bit with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_segments_max_unsafe_auto_id_timestamp` | GaugeValue | Current max unsafe auto id timestamp with all shards on all nodes | +| `elasticsearch_indices_stats_total_translog_earliest_last_modified_age` | GaugeValue | Current earliest last modified age with all shards on all nodes | +| `elasticsearch_indices_stats_total_translog_operations` | GaugeValue | Current number of operations in the transaction log with all shards on all nodes | +| `elasticsearch_indices_stats_total_translog_size_in_bytes` | GaugeValue | Current size of transaction log with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_translog_uncommitted_operations` | GaugeValue | Current number of uncommitted operations in the transaction log with all shards on all nodes | +| `elasticsearch_indices_stats_total_translog_uncommitted_size_in_bytes` | GaugeValue | Current size of uncommitted transaction log with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_completion_size_in_bytes` | GaugeValue | Current size of completion with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_total_search_query_time_seconds` | CounterValue | Total search query time in seconds | +| `elasticsearch_indices_stats_total_search_query_current` | GaugeValue | The number of currently active queries | +| `elasticsearch_indices_stats_total_search_open_contexts` | CounterValue | Total number of open search contexts | +| `elasticsearch_indices_stats_total_search_query_total` | CounterValue | Total number of queries | +| `elasticsearch_indices_stats_total_search_fetch_time_seconds` | CounterValue | Total search fetch time in seconds | +| `elasticsearch_indices_stats_total_search_fetch` | CounterValue | Total search fetch count | +| `elasticsearch_indices_stats_total_search_fetch_current` | CounterValue | Current search fetch count | +| `elasticsearch_indices_stats_total_search_scroll_time_seconds` | CounterValue | Total search scroll time in seconds | +| `elasticsearch_indices_stats_total_search_scroll_current` | GaugeValue | Current search scroll count | +| `elasticsearch_indices_stats_total_search_scroll` | CounterValue | Total search scroll count | +| `elasticsearch_indices_stats_total_search_suggest_time_seconds` | CounterValue | Total search suggest time in seconds | +| `elasticsearch_indices_stats_total_search_suggest_total` | CounterValue | Total search suggest count | +| `elasticsearch_indices_stats_total_search_suggest_current` | CounterValue | Current search suggest count | +| `elasticsearch_indices_stats_total_indexing_index_time_seconds` | CounterValue | Total indexing index time in seconds | +| `elasticsearch_indices_stats_total_index_current` | GaugeValue | The number of documents currently being indexed | +| `elasticsearch_indices_stats_total_index_failed` | GaugeValue | Total indexing index failed count | +| `elasticsearch_indices_stats_total_delete_current` | GaugeValue | The number of delete operations currently being processed | +| `elasticsearch_indices_stats_total_indexing_index` | CounterValue | Total indexing index count | +| `elasticsearch_indices_stats_total_indexing_delete_time_seconds` | CounterValue | Total indexing delete time in seconds | +| `elasticsearch_indices_stats_total_indexing_delete` | CounterValue | Total indexing delete count | +| `elasticsearch_indices_stats_total_indexing_noop_update` | CounterValue | Total indexing no-op update count | +| `elasticsearch_indices_stats_total_indexing_throttle_time_seconds` | CounterValue | Total indexing throttle time in seconds | +| `elasticsearch_indices_stats_total_get_time_seconds` | CounterValue | Total get time in seconds | +| `elasticsearch_indices_stats_total_get_exists_total` | CounterValue | Total exists count | +| `elasticsearch_indices_stats_total_get_exists_time_seconds` | CounterValue | Total exists time in seconds | +| `elasticsearch_indices_stats_total_get_total` | CounterValue | Total get count | +| `elasticsearch_indices_stats_total_get_missing_total` | CounterValue | Total missing count | +| `elasticsearch_indices_stats_total_get_missing_time_seconds` | CounterValue | Total missing time in seconds | +| `elasticsearch_indices_stats_total_get_current` | CounterValue | Current get count | +| `elasticsearch_indices_stats_total_merges_time_seconds` | CounterValue | Total merge time in seconds | +| `elasticsearch_indices_stats_total_merges_total` | CounterValue | Total merge count | +| `elasticsearch_indices_stats_total_merges_total_docs` | CounterValue | Total merge docs count | +| `elasticsearch_indices_stats_total_merges_total_size_in_bytes` | CounterValue | Total merge size in bytes | +| `elasticsearch_indices_stats_total_merges_current` | CounterValue | Current merge count | +| `elasticsearch_indices_stats_total_merges_current_docs` | CounterValue | Current merge docs count | +| `elasticsearch_indices_stats_total_merges_current_size_in_bytes` | CounterValue | Current merge size in bytes | +| `elasticsearch_indices_stats_total_merges_total_throttle_time_seconds` | CounterValue | Total merge I/O throttle time in seconds | +| `elasticsearch_indices_stats_total_merges_total_stopped_time_seconds` | CounterValue | Total large merge stopped time in seconds, allowing smaller merges to complete | +| `elasticsearch_indices_stats_total_merges_total_auto_throttle_bytes` | CounterValue | Total bytes that were auto-throttled during merging | +| `elasticsearch_indices_stats_total_refresh_external_total_time_seconds` | CounterValue | Total external refresh time in seconds | +| `elasticsearch_indices_stats_total_refresh_external_total` | CounterValue | Total external refresh count | +| `elasticsearch_indices_stats_total_refresh_total_time_seconds` | CounterValue | Total refresh time in seconds | +| `elasticsearch_indices_stats_total_refresh_total` | CounterValue | Total refresh count | +| `elasticsearch_indices_stats_total_refresh_listeners` | CounterValue | Total number of refresh listeners | +| `elasticsearch_indices_stats_total_recovery_current_as_source` | CounterValue | Current number of recovery as source | +| `elasticsearch_indices_stats_total_recovery_current_as_target` | CounterValue | Current number of recovery as target | +| `elasticsearch_indices_stats_total_recovery_throttle_time_seconds` | CounterValue | Total recovery throttle time in seconds | +| `elasticsearch_indices_stats_total_flush_time_seconds_total` | CounterValue | Total flush time in seconds | +| `elasticsearch_indices_stats_total_flush_total` | CounterValue | Total flush count | +| `elasticsearch_indices_stats_total_flush_periodic` | CounterValue | Total periodic flush count | +| `elasticsearch_indices_stats_total_warmer_time_seconds_total` | CounterValue | Total warmer time in seconds | +| `elasticsearch_indices_stats_total_warmer_total` | CounterValue | Total warmer count | +| `elasticsearch_indices_stats_total_query_cache_memory_in_bytes` | CounterValue | Total query cache memory bytes | +| `elasticsearch_indices_stats_total_query_cache_size` | GaugeValue | Total query cache size | +| `elasticsearch_indices_stats_total_query_cache_total_count` | CounterValue | Total query cache count | +| `elasticsearch_indices_stats_total_query_cache_hit_count` | CounterValue | Total query cache hits count | +| `elasticsearch_indices_stats_total_query_cache_miss_count` | CounterValue | Total query cache misses count | +| `elasticsearch_indices_stats_total_query_cache_cache_count` | CounterValue | Total query cache caches count | +| `elasticsearch_indices_stats_total_query_cache_evictions` | CounterValue | Total query cache evictions count | +| `elasticsearch_indices_stats_total_request_cache_memory_in_bytes` | CounterValue | Total request cache memory bytes | +| `elasticsearch_indices_stats_total_request_cache_hit_count` | CounterValue | Total request cache hits count | +| `elasticsearch_indices_stats_total_request_cache_miss_count` | CounterValue | Total request cache misses count | +| `elasticsearch_indices_stats_total_request_cache_evictions` | CounterValue | Total request cache evictions count | +| `elasticsearch_indices_stats_total_fielddata_memory_in_bytes` | CounterValue | Total fielddata memory bytes | +| `elasticsearch_indices_stats_total_fielddata_evictions` | CounterValue | Total fielddata evictions count | +| `elasticsearch_indices_stats_total_seq_no_global_checkpoint` | CounterValue | Global checkpoint | +| `elasticsearch_indices_stats_total_seq_no_local_checkpoint` | CounterValue | Local checkpoint | +| `elasticsearch_indices_stats_total_seq_no_max_seq_no` | CounterValue | Max sequence number | + + +| Name | Type | Description | +|---------------------------------------------------------------------------------|--------------|----------------------------------------------------------------------------------------------| +| `elasticsearch_indices_stats_primaries_docs_count` | GaugeValue | Total count of documents | +| `elasticsearch_indices_stats_primaries_docs_deleted` | GaugeValue | Total count of deleted documents | +| `elasticsearch_indices_stats_primaries_store_size_in_bytes` | GaugeValue | Current total size of stored index data in bytes with all shards on all nodes | +| `elasticsearch_indices_stats_primaries_throttle_time_seconds` | GaugeValue | Total time the index has been throttled in seconds | +| `elasticsearch_indices_stats_primaries_segments_count` | GaugeValue | Current number of segments with all shards on all nodes | +| `elasticsearch_indices_stats_primaries_segments_memory_in_bytes` | GaugeValue | Current size of segments with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_terms_memory_in_bytes` | GaugeValue | Current number of terms with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_stored_fields_memory_in_bytes` | GaugeValue | Current size of fields with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_term_vectors_memory_in_bytes` | GaugeValue | Current size of term vectors with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_norms_memory_in_bytes` | GaugeValue | Current size of norms with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_points_memory_in_bytes` | GaugeValue | Current size of points with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_doc_values_memory_in_bytes` | GaugeValue | Current size of doc values with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_index_writer_memory_in_bytes` | GaugeValue | Current size of index writer with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_version_map_memory_in_bytes` | GaugeValue | Current size of version map with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_fixed_bit_set_memory_in_bytes` | GaugeValue | Current size of fixed bit with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_segments_max_unsafe_auto_id_timestamp` | GaugeValue | Current max unsafe auto id timestamp with all shards on all nodes | +| `elasticsearch_indices_stats_primaries_translog_earliest_last_modified_age` | GaugeValue | Current earliest last modified age with all shards on all nodes | +| `elasticsearch_indices_stats_primaries_translog_operations` | GaugeValue | Current number of operations in the transaction log with all shards on all nodes | +| `elasticsearch_indices_stats_primaries_translog_size_in_bytes` | GaugeValue | Current size of transaction log with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_translog_uncommitted_operations` | GaugeValue | Current number of uncommitted operations in the transaction log with all shards on all nodes | +| `elasticsearch_indices_stats_primaries_translog_uncommitted_size_in_bytes` | GaugeValue | Current size of uncommitted transaction log with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_completion_size_in_bytes` | GaugeValue | Current size of completion with all shards on all nodes in bytes | +| `elasticsearch_indices_stats_primaries_search_query_time_seconds` | CounterValue | Total search query time in seconds | +| `elasticsearch_indices_stats_primaries_search_query_current` | GaugeValue | The number of currently active queries | +| `elasticsearch_indices_stats_primaries_search_open_contexts` | CounterValue | Total number of open search contexts | +| `elasticsearch_indices_stats_primaries_search_query_total` | CounterValue | Total number of queries | +| `elasticsearch_indices_stats_primaries_search_fetch_time_seconds` | CounterValue | Total search fetch time in seconds | +| `elasticsearch_indices_stats_primaries_search_fetch` | CounterValue | Total search fetch count | +| `elasticsearch_indices_stats_primaries_search_fetch_current` | CounterValue | Current search fetch count | +| `elasticsearch_indices_stats_primaries_search_scroll_time_seconds` | CounterValue | Total search scroll time in seconds | +| `elasticsearch_indices_stats_primaries_search_scroll_current` | GaugeValue | Current search scroll count | +| `elasticsearch_indices_stats_primaries_search_scroll` | CounterValue | Total search scroll count | +| `elasticsearch_indices_stats_primaries_search_suggest_time_seconds` | CounterValue | Total search suggest time in seconds | +| `elasticsearch_indices_stats_primaries_search_suggest_total` | CounterValue | Total search suggest count | +| `elasticsearch_indices_stats_primaries_search_suggest_current` | CounterValue | Current search suggest count | +| `elasticsearch_indices_stats_primaries_indexing_index_time_seconds` | CounterValue | Total indexing index time in seconds | +| `elasticsearch_indices_stats_primaries_index_current` | GaugeValue | The number of documents currently being indexed | +| `elasticsearch_indices_stats_primaries_index_failed` | GaugeValue | Total indexing index failed count | +| `elasticsearch_indices_stats_primaries_delete_current` | GaugeValue | The number of delete operations currently being processed | +| `elasticsearch_indices_stats_primaries_indexing_index` | CounterValue | Total indexing index count | +| `elasticsearch_indices_stats_primaries_indexing_delete_time_seconds` | CounterValue | Total indexing delete time in seconds | +| `elasticsearch_indices_stats_primaries_indexing_delete` | CounterValue | Total indexing delete count | +| `elasticsearch_indices_stats_primaries_indexing_noop_update` | CounterValue | Total indexing no-op update count | +| `elasticsearch_indices_stats_primaries_indexing_throttle_time_seconds` | CounterValue | Total indexing throttle time in seconds | +| `elasticsearch_indices_stats_primaries_get_time_seconds` | CounterValue | Total get time in seconds | +| `elasticsearch_indices_stats_primaries_get_exists_total` | CounterValue | Total exists count | +| `elasticsearch_indices_stats_primaries_get_exists_time_seconds` | CounterValue | Total exists time in seconds | +| `elasticsearch_indices_stats_primaries_get_total` | CounterValue | Total get count | +| `elasticsearch_indices_stats_primaries_get_missing_total` | CounterValue | Total missing count | +| `elasticsearch_indices_stats_primaries_get_missing_time_seconds` | CounterValue | Total missing time in seconds | +| `elasticsearch_indices_stats_primaries_get_current` | CounterValue | Current get count | +| `elasticsearch_indices_stats_primaries_merges_time_seconds` | CounterValue | Total merge time in seconds | +| `elasticsearch_indices_stats_primaries_merges_total` | CounterValue | Total merge count | +| `elasticsearch_indices_stats_primaries_merges_primaries_docs` | CounterValue | Total merge docs count | +| `elasticsearch_indices_stats_primaries_merges_primaries_size_in_bytes` | CounterValue | Total merge size in bytes | +| `elasticsearch_indices_stats_primaries_merges_current` | CounterValue | Current merge count | +| `elasticsearch_indices_stats_primaries_merges_current_docs` | CounterValue | Current merge docs count | +| `elasticsearch_indices_stats_primaries_merges_current_size_in_bytes` | CounterValue | Current merge size in bytes | +| `elasticsearch_indices_stats_primaries_merges_primaries_throttle_time_seconds` | CounterValue | Total merge I/O throttle time in seconds | +| `elasticsearch_indices_stats_primaries_merges_primaries_stopped_time_seconds` | CounterValue | Total large merge stopped time in seconds, allowing smaller merges to complete | +| `elasticsearch_indices_stats_primaries_merges_primaries_auto_throttle_bytes` | CounterValue | Total bytes that were auto-throttled during merging | +| `elasticsearch_indices_stats_primaries_refresh_external_primaries_time_seconds` | CounterValue | Total external refresh time in seconds | +| `elasticsearch_indices_stats_primaries_refresh_external_total` | CounterValue | Total external refresh count | +| `elasticsearch_indices_stats_primaries_refresh_primaries_time_seconds` | CounterValue | Total refresh time in seconds | +| `elasticsearch_indices_stats_primaries_refresh_total` | CounterValue | Total refresh count | +| `elasticsearch_indices_stats_primaries_refresh_listeners` | CounterValue | Total number of refresh listeners | +| `elasticsearch_indices_stats_primaries_recovery_current_as_source` | CounterValue | Current number of recovery as source | +| `elasticsearch_indices_stats_primaries_recovery_current_as_target` | CounterValue | Current number of recovery as target | +| `elasticsearch_indices_stats_primaries_recovery_throttle_time_seconds` | CounterValue | Total recovery throttle time in seconds | +| `elasticsearch_indices_stats_primaries_flush_time_seconds_total` | CounterValue | Total flush time in seconds | +| `elasticsearch_indices_stats_primaries_flush_total` | CounterValue | Total flush count | +| `elasticsearch_indices_stats_primaries_flush_periodic` | CounterValue | Total periodic flush count | +| `elasticsearch_indices_stats_primaries_warmer_time_seconds_total` | CounterValue | Total warmer time in seconds | +| `elasticsearch_indices_stats_primaries_warmer_total` | CounterValue | Total warmer count | +| `elasticsearch_indices_stats_primaries_query_cache_memory_in_bytes` | CounterValue | Total query cache memory bytes | +| `elasticsearch_indices_stats_primaries_query_cache_size` | GaugeValue | Total query cache size | +| `elasticsearch_indices_stats_primaries_query_cache_primaries_count` | CounterValue | Total query cache count | +| `elasticsearch_indices_stats_primaries_query_cache_hit_count` | CounterValue | Total query cache hits count | +| `elasticsearch_indices_stats_primaries_query_cache_miss_count` | CounterValue | Total query cache misses count | +| `elasticsearch_indices_stats_primaries_query_cache_cache_count` | CounterValue | Total query cache caches count | +| `elasticsearch_indices_stats_primaries_query_cache_evictions` | CounterValue | Total query cache evictions count | +| `elasticsearch_indices_stats_primaries_request_cache_memory_in_bytes` | CounterValue | Total request cache memory bytes | +| `elasticsearch_indices_stats_primaries_request_cache_hit_count` | CounterValue | Total request cache hits count | +| `elasticsearch_indices_stats_primaries_request_cache_miss_count` | CounterValue | Total request cache misses count | +| `elasticsearch_indices_stats_primaries_request_cache_evictions` | CounterValue | Total request cache evictions count | +| `elasticsearch_indices_stats_primaries_fielddata_memory_in_bytes` | CounterValue | Total fielddata memory bytes | +| `elasticsearch_indices_stats_primaries_fielddata_evictions` | CounterValue | Total fielddata evictions count | +| `elasticsearch_indices_stats_primaries_seq_no_global_checkpoint` | CounterValue | Global checkpoint | +| `elasticsearch_indices_stats_primaries_seq_no_local_checkpoint` | CounterValue | Local checkpoint | +| `elasticsearch_indices_stats_primaries_seq_no_max_seq_no` | CounterValue | Max sequence number | + +#### `allnodes = true` + +| Name | Type | Description | +|----------------------------------------------------------------|--------------|--------------------------------------------------------| +| `elasticsearch_os_cpu_load_average_1m` | GaugeValue | Shortterm load average | +| `elasticsearch_os_cpu_load_average_5m` | GaugeValue | Midterm load average | +| `elasticsearch_os_cpu_load_average_15m` | GaugeValue | Longterm load average | +| `elasticsearch_os_cpu_percent` | GaugeValue | Percent CPU used by OS | +| `elasticsearch_os_mem_free_in_bytes` | GaugeValue | Amount of free physical memory in bytes | +| `elasticsearch_os_mem_used_in_bytes` | GaugeValue | Amount of used physical memory in bytes | +| `elasticsearch_os_mem_actual_free_in_bytes` | GaugeValue | Amount of free physical memory in bytes | +| `elasticsearch_os_mem_actual_used_in_bytes` | GaugeValue | Amount of used physical memory in bytes | +| `elasticsearch_os_mem_used_percent` | GaugeValue | Percent of used physical memory | +| `elasticsearch_os_mem_total_in_bytes` | GaugeValue | Amount of used physical memory in bytes | +| `elasticsearch_os_mem_free_percent` | GaugeValue | Percent of free physical memory | +| `elasticsearch_os_cgroup_cpu_cfs_period_micros` | GaugeValue | CPU CFS period in microseconds | +| `elasticsearch_os_cgroup_cpu_cfs_quota_micros` | GaugeValue | CPU CFS quota in microseconds | +| `elasticsearch_os_cgroup_cpu_stat_number_of_elapsed_periods` | GaugeValue | CPU CFS quota in microseconds | +| `elasticsearch_os_cgroup_cpu_stat_number_of_times_throttled` | GaugeValue | CPU CFS quota in microseconds | +| `elasticsearch_os_cgroup_cpu_stat_time_throttled_nanos` | GaugeValue | CPU CFS quota in microseconds | +| `elasticsearch_os_cgroup_cpuacct_usage_nanos` | GaugeValue | Cpuacct usage in nanos | +| `elasticsearch_os_swap_used_in_bytes` | GaugeValue | Amount of used swap memory in bytes | +| `elasticsearch_os_swap_total_in_bytes` | GaugeValue | Amount of total swap memory in bytes | +| `elasticsearch_os_swap_free_in_bytes` | GaugeValue | Amount of free swap memory in bytes | +| `elasticsearch_indices_fielddata_memory_size_in_bytes` | GaugeValue | Field data cache memory usage in bytes | +| `elasticsearch_indices_fielddata_evictions` | CounterValue | Evictions from field data | +| `elasticsearch_indices_completion_size_in_bytes` | CounterValue | Completion in bytes | +| `elasticsearch_indices_filter_cache_memory_size_in_bytes` | GaugeValue | Filter cache memory usage in bytes | +| `elasticsearch_indices_filter_cache_evictions` | CounterValue | Evictions from filter cache | +| `elasticsearch_indices_query_cache_memory_size_in_bytes` | GaugeValue | Query cache memory usage in bytes | +| `elasticsearch_indices_query_cache_evictions` | CounterValue | Evictions from query cache | +| `elasticsearch_indices_query_cache_total_count` | CounterValue | Query cache total count | +| `elasticsearch_indices_query_cache_cache_size` | GaugeValue | Query cache cache size | +| `elasticsearch_indices_query_cache_cache_count` | CounterValue | Query cache cache count | +| `elasticsearch_indices_query_cache_hit_count` | CounterValue | Query cache hit count | +| `elasticsearch_indices_query_cache_miss_count` | CounterValue | Query cache miss count | +| `elasticsearch_indices_request_cache_memory_size_in_bytes` | GaugeValue | Request cache memory usage in bytes | +| `elasticsearch_indices_request_cache_evictions` | CounterValue | Evictions from request cache | +| `elasticsearch_indices_request_cache_hit_count` | CounterValue | Request cache hit count | +| `elasticsearch_indices_request_cache_miss_count` | CounterValue | Request cache miss count | +| `elasticsearch_indices_translog_operations` | CounterValue | Total translog operations | +| `elasticsearch_indices_translog_size_in_bytes` | GaugeValue | Total translog size in bytes | +| `elasticsearch_indices_get_time_seconds` | CounterValue | Total get time in seconds | +| `elasticsearch_indices_get_total` | CounterValue | Total get count | +| `elasticsearch_indices_get_missing_time_seconds` | CounterValue | Total time of get missing in seconds | +| `elasticsearch_indices_get_missing_total` | CounterValue | Total get missing count | +| `elasticsearch_indices_get_exists_time_seconds` | CounterValue | Total time get exists in seconds | +| `elasticsearch_indices_get_exists_total` | CounterValue | Total get exists operations | +| `elasticsearch_indices_refresh_time_seconds_total` | CounterValue | Total time spent refreshing in seconds | +| `elasticsearch_indices_refresh_total` | CounterValue | Total refreshes | +| `elasticsearch_indices_search_query_time_seconds` | CounterValue | Total search query time in seconds | +| `elasticsearch_indices_search_query_total` | CounterValue | Total number of queries | +| `elasticsearch_indices_search_fetch_time_seconds` | CounterValue | Total search fetch time in seconds | +| `elasticsearch_indices_search_fetch_total` | CounterValue | Total search fetch count | +| `elasticsearch_indices_search_suggest_total` | CounterValue | Total search suggest count | +| `elasticsearch_indices_search_suggest_time_seconds` | CounterValue | Total search suggest time in seconds | +| `elasticsearch_indices_search_scroll_total` | CounterValue | Total search scroll count | +| `elasticsearch_indices_search_scroll_time_seconds` | CounterValue | Total search scroll time in seconds | +| `elasticsearch_indices_docs_count` | GaugeValue | Count of documents on this node | +| `elasticsearch_indices_docs_deleted` | GaugeValue | Count of deleted documents on this node | +| `elasticsearch_indices_store_size_in_bytes` | GaugeValue | Current size of stored index data | +| `elasticsearch_indices_merges_total_size_in_bytes` | CounterValue | Total merge size in bytes | +| `elasticsearch_indices_merges_total_time_seconds_total` | CounterValue | Total time spent merging in seconds | +| `elasticsearch_indices_merges_total_throttled_time_seconds` | CounterValue | Total throttled time of merges in seconds | +| `elasticsearch_jvm_threads_count` | GaugeValue | Count of threads | +| `elasticsearch_jvm_threads_peak_count` | GaugeValue | Peak count of threads | +| `elasticsearch_jvm_timestamp` | GaugeValue | JVM timestamp | +| `elasticsearch_jvm_mem_heap_used_in_bytes` | GaugeValue | JVM memory currently used by heap in bytes | +| `elasticsearch_jvm_mem_non_heap_used_in_bytes` | GaugeValue | JVM memory currently used by non-heap in bytes | +| `elasticsearch_jvm_mem_heap_max_in_bytes` | GaugeValue | Maximum JVM heap memory in bytes | +| `elasticsearch_jvm_mem_heap_used_percent` | GaugeValue | JVM heap memory used percent | +| `elasticsearch_jvm_mem_heap_committed_in_bytes` | GaugeValue | JVM heap memory committed in bytes | +| `elasticsearch_jvm_mem_non_heap_committed_in_bytes` | GaugeValue | JVM non-heap memory committed in bytes | +| `elasticsearch_jvm_memory_pools_young_used_in_bytes` | GaugeValue | JVM young generation pool used memory in bytes | +| `elasticsearch_jvm_memory_pools_young_max_in_bytes` | CounterValue | Maximum JVM young generation pool memory in bytes | +| `elasticsearch_jvm_memory_pools_young_peak_used_in_bytes` | CounterValue | Peak used JVM young generation pool memory in bytes | +| `elasticsearch_jvm_memory_pools_young_peak_max_in_bytes` | CounterValue | Peak maximum JVM young generation pool memory in bytes | +| `elasticsearch_jvm_memory_pools_survivor_used_in_bytes` | GaugeValue | JVM survivor space pool used memory in bytes | +| `elasticsearch_jvm_memory_pools_survivor_max_in_bytes` | CounterValue | Maximum JVM survivor space pool memory in bytes | +| `elasticsearch_jvm_memory_pools_survivor_peak_used_in_bytes` | CounterValue | Peak used JVM survivor space pool memory in bytes | +| `elasticsearch_jvm_memory_pools_survivor_peak_max_in_bytes` | CounterValue | Peak maximum JVM survivor space pool memory in bytes | +| `elasticsearch_jvm_memory_pools_old_used_in_bytes` | GaugeValue | JVM old generation pool used memory in bytes | +| `elasticsearch_jvm_memory_pools_old_max_in_bytes` | CounterValue | Maximum JVM old generation pool memory in bytes | +| `elasticsearch_jvm_memory_pools_old_peak_used_in_bytes` | CounterValue | Peak used JVM old generation pool memory in bytes | +| `elasticsearch_jvm_memory_pools_old_peak_max_in_bytes` | CounterValue | Peak maximum JVM old generation pool memory in bytes | +| `elasticsearch_jvm_buffer_pool_direct_count` | GaugeValue | JVM buffer pool direct count | +| `elasticsearch_jvm_buffer_pool_direct_total_capacity_in_bytes` | GaugeValue | JVM buffer pool direct total capacity in bytes | +| `elasticsearch_jvm_buffer_pool_direct_used_in_bytes` | GaugeValue | JVM buffer pool direct used in bytes | +| `elasticsearch_jvm_buffer_pool_mapped_count` | GaugeValue | JVM buffer pool mapped count | +| `elasticsearch_jvm_buffer_pool_mapped_total_capacity_in_bytes` | GaugeValue | JVM buffer pool mapped total capacity in bytes | +| `elasticsearch_jvm_buffer_pool_mapped_used_in_bytes` | GaugeValue | JVM buffer pool mapped used in bytes | +| `elasticsearch_jvm_classes_current_loaded_count` | GaugeValue | JVM classes currently loaded count | +| `elasticsearch_jvm_classes_total_loaded_count` | GaugeValue | JVM classes total loaded count | +| `elasticsearch_jvm_classes_total_unloaded_count` | GaugeValue | JVM classes total unloaded count | +| `elasticsearch_jvm_uptime_seconds` | GaugeValue | JVM process uptime in seconds | +| `elasticsearch_process_cpu_percent` | GaugeValue | Percent CPU used by process | +| `elasticsearch_process_mem_resident_size_in_bytes` | GaugeValue | Resident memory in use by process in bytes | +| `elasticsearch_process_mem_share_size_in_bytes` | GaugeValue | Shared memory in use by process in bytes | + +#### `export_indices_settings = true` + +| Name | Type | Help | +|----------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------| +| elasticsearch_indices_settings_creation_timestamp_seconds | gauge | Timestamp of the index creation in seconds | +| elasticsearch_indices_settings_stats_read_only_indices | gauge | Count of indices that have read_only_allow_delete=true | +| elasticsearch_indices_settings_total_fields | gauge | Index setting value for index.mapping.total_fields.limit (total allowable mapped fields in a index) | +| elasticsearch_indices_settings_replicas | gauge | Index setting value for index.replicas | + +#### `export_indices_mappings = true` + +| Name | Type | Help | +|----------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------| +| elasticsearch_indices_mappings_stats_fields | gauge | Count of fields currently mapped by index | +| elasticsearch_indices_mappings_stats_json_parse_failures_total | counter | Number of errors while parsing JSON | +| elasticsearch_indices_mappings_stats_scrapes_total | counter | Current total Elasticsearch Indices Mappings scrapes | +| elasticsearch_indices_mappings_stats_up | gauge | Was the last scrape of the Elasticsearch Indices Mappings endpoint successful | + +#### `export_slm = true` + +| Name | Type | Help | +|----------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------| +| elasticsearch_slm_stats_up | gauge | Up metric for SLM collector | +| elasticsearch_slm_stats_total_scrapes | counter | Number of scrapes for SLM collector | +| elasticsearch_slm_stats_json_parse_failures | counter | JSON parse failures for SLM collector | +| elasticsearch_slm_stats_retention_runs_total | counter | Total retention runs | +| elasticsearch_slm_stats_retention_failed_total | counter | Total failed retention runs | +| elasticsearch_slm_stats_retention_timed_out_total | counter | Total retention run timeouts | +| elasticsearch_slm_stats_retention_deletion_time_seconds | gauge | Retention run deletion time | +| elasticsearch_slm_stats_total_snapshots_taken_total | counter | Total snapshots taken | +| elasticsearch_slm_stats_total_snapshots_failed_total | counter | Total snapshots failed | +| elasticsearch_slm_stats_total_snapshots_deleted_total | counter | Total snapshots deleted | +| elasticsearch_slm_stats_total_snapshots_failed_total | counter | Total snapshots failed | +| elasticsearch_slm_stats_snapshots_taken_total | counter | Snapshots taken by policy | +| elasticsearch_slm_stats_snapshots_failed_total | counter | Snapshots failed by policy | +| elasticsearch_slm_stats_snapshots_deleted_total | counter | Snapshots deleted by policy | +| elasticsearch_slm_stats_snapshot_deletion_failures_total | counter | Snapshot deletion failures by policy | +| elasticsearch_slm_stats_operation_mode | gauge | SLM operation mode (Running, stopping, stopped) | diff --git a/inputs/elasticsearch/collector/cluster_health.go b/inputs/elasticsearch/collector/cluster_health.go new file mode 100644 index 00000000..27a8a57b --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_health.go @@ -0,0 +1,281 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + colors = []string{"green", "yellow", "red"} + defaultClusterHealthLabels = []string{"cluster"} +) + +type clusterHealthMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(clusterHealth clusterHealthResponse) float64 +} + +type clusterHealthStatusMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(clusterHealth clusterHealthResponse, color string) float64 + Labels func(clusterName, color string) []string +} + +// ClusterHealth type defines the collector struct +type ClusterHealth struct { + client *http.Client + url *url.URL + + metrics []*clusterHealthMetric + statusMetric *clusterHealthStatusMetric +} + +// NewClusterHealth returns a new Collector exposing ClusterHealth stats. +func NewClusterHealth(client *http.Client, url *url.URL) *ClusterHealth { + subsystem := "cluster_health" + + return &ClusterHealth{ + client: client, + url: url, + + metrics: []*clusterHealthMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "active_primary_shards"), + "The number of primary shards in your cluster. This is an aggregate total across all indices.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.ActivePrimaryShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "active_shards"), + "Aggregate total of all shards across all indices, which includes replica shards.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.ActiveShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "active_shards_percent"), + "Percentage of active shards in the cluster.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return clusterHealth.ActiveShardsPercentAsNumber + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "delayed_unassigned_shards"), + "Shards delayed to reduce reallocation overhead", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.DelayedUnassignedShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "initializing_shards"), + "Count of shards that are being freshly created.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.InitializingShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "number_of_data_nodes"), + "Number of data nodes in the cluster.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.NumberOfDataNodes) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "number_of_in_flight_fetch"), + "The number of ongoing shard info requests.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.NumberOfInFlightFetch) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "task_max_waiting_in_queue_millis"), + "Tasks max time waiting in queue.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.TaskMaxWaitingInQueueMillis) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "number_of_nodes"), + "Number of nodes in the cluster.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.NumberOfNodes) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "number_of_pending_tasks"), + "Cluster level changes which have not yet been executed", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.NumberOfPendingTasks) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "relocating_shards"), + "The number of shards that are currently moving from one node to another node.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.RelocatingShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "unassigned_shards"), + "The number of shards that exist in the cluster state, but cannot be found in the cluster itself.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.UnassignedShards) + }, + }, + }, + statusMetric: &clusterHealthStatusMetric{ + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "status"), + "Whether all primary and replica shards are allocated.", + []string{"cluster", "color"}, nil, + ), + Value: func(clusterHealth clusterHealthResponse, color string) float64 { + if clusterHealth.Status == color { + return 1 + } + return 0 + }, + }, + } +} + +// Describe set Prometheus metrics descriptions. +func (c *ClusterHealth) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range c.metrics { + ch <- metric.Desc + } + ch <- c.statusMetric.Desc +} + +func (c *ClusterHealth) fetchAndDecodeClusterHealth() (clusterHealthResponse, error) { + var chr clusterHealthResponse + + u := *c.url + u.Path = path.Join(u.Path, "/_cluster/health") + res, err := c.client.Get(u.String()) + if err != nil { + return chr, fmt.Errorf("failed to get cluster health from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return chr, err + } + + if err := json.Unmarshal(bts, &chr); err != nil { + return chr, err + } + + return chr, nil +} + +// Collect collects ClusterHealth metrics. +func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric) { + clusterHealthResp, err := c.fetchAndDecodeClusterHealth() + if err != nil { + log.Println("failed to fetch and decode cluster health, err: ", err) + return + } + + for _, metric := range c.metrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(clusterHealthResp), + clusterHealthResp.ClusterName, + ) + } + + for _, color := range colors { + ch <- prometheus.MustNewConstMetric( + c.statusMetric.Desc, + c.statusMetric.Type, + c.statusMetric.Value(clusterHealthResp, color), + clusterHealthResp.ClusterName, color, + ) + } +} diff --git a/inputs/elasticsearch/collector/cluster_health_indices.go b/inputs/elasticsearch/collector/cluster_health_indices.go new file mode 100644 index 00000000..fb6ae3a1 --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_health_indices.go @@ -0,0 +1,198 @@ +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +// ClusterHealthIndices type defines the collector struct +type ClusterHealthIndices struct { + client *http.Client + url *url.URL + + metrics []*clusterHealthMetric + statusMetric *clusterHealthStatusMetric +} + +// NewClusterHealthIndices returns a new Collector exposing ClusterHealth stats. +func NewClusterHealthIndices(client *http.Client, url *url.URL) *ClusterHealth { + subsystem := "cluster_health_indices" + + return &ClusterHealth{ + client: client, + url: url, + + metrics: []*clusterHealthMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "active_primary_shards"), + "The number of primary shards in your cluster. This is an aggregate total across all indices.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.ActivePrimaryShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "active_shards"), + "Aggregate total of all shards across all indices, which includes replica shards.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.ActiveShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "initializing_shards"), + "Count of shards that are being freshly created.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.InitializingShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "number_of_replicas"), + "Number of replicas in the cluster.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.NumberOfReplicas) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "number_of_shards"), + "Number of shards in the cluster.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.NumberOfShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "relocating_shards"), + "The number of shards that are currently moving from one node to another node.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.RelocatingShards) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "unassigned_shards"), + "The number of shards that exist in the cluster state, but cannot be found in the cluster itself.", + defaultClusterHealthLabels, nil, + ), + Value: func(clusterHealth clusterHealthResponse) float64 { + return float64(clusterHealth.UnassignedShards) + }, + }, + }, + statusMetric: &clusterHealthStatusMetric{ + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "status"), + "Whether all primary and replica shards are allocated.", + []string{"cluster", "color"}, nil, + ), + Value: func(clusterHealth clusterHealthResponse, color string) float64 { + if clusterHealth.Status == color { + return 1 + } + return 0 + }, + }, + } +} + +// Describe set Prometheus metrics descriptions. +func (c *ClusterHealthIndices) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range c.metrics { + ch <- metric.Desc + } + ch <- c.statusMetric.Desc +} + +func (c *ClusterHealthIndices) fetchAndDecodeClusterHealthIndices() (clusterHealthResponse, error) { + var chr clusterHealthResponse + + u := *c.url + u.Path = path.Join(u.Path, "/_cluster/health") + v := url.Values{} + v.Add("level", "indices") + u.RawQuery = v.Encode() + res, err := c.client.Get(u.String()) + if err != nil { + return chr, fmt.Errorf("failed to get cluster health indicies from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return chr, err + } + + if err := json.Unmarshal(bts, &chr); err != nil { + return chr, err + } + + return chr, nil +} + +// Collect collects ClusterHealth metrics. +func (c *ClusterHealthIndices) Collect(ch chan<- prometheus.Metric) { + clusterHealthResp, err := c.fetchAndDecodeClusterHealthIndices() + if err != nil { + log.Println("failed to fetch and decode cluster health, err: ", err) + return + } + + for _, metric := range c.metrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(clusterHealthResp), + clusterHealthResp.ClusterName, + ) + } + + for _, color := range colors { + ch <- prometheus.MustNewConstMetric( + c.statusMetric.Desc, + c.statusMetric.Type, + c.statusMetric.Value(clusterHealthResp, color), + clusterHealthResp.ClusterName, color, + ) + } +} diff --git a/inputs/elasticsearch/collector/cluster_health_response.go b/inputs/elasticsearch/collector/cluster_health_response.go new file mode 100644 index 00000000..a095df15 --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_health_response.go @@ -0,0 +1,34 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +type clusterHealthResponse struct { + ClusterName string `json:"cluster_name"` + Status string `json:"status"` + TimedOut bool `json:"timed_out"` + NumberOfNodes int `json:"number_of_nodes"` + NumberOfDataNodes int `json:"number_of_data_nodes"` + NumberOfReplicas int `json:"number_of_replicas"` + NumberOfShards int `json:"number_of_shards"` + ActivePrimaryShards int `json:"active_primary_shards"` + ActiveShards int `json:"active_shards"` + RelocatingShards int `json:"relocating_shards"` + InitializingShards int `json:"initializing_shards"` + UnassignedShards int `json:"unassigned_shards"` + DelayedUnassignedShards int `json:"delayed_unassigned_shards"` + NumberOfPendingTasks int `json:"number_of_pending_tasks"` + NumberOfInFlightFetch int `json:"number_of_in_flight_fetch"` + TaskMaxWaitingInQueueMillis int `json:"task_max_waiting_in_queue_millis"` + ActiveShardsPercentAsNumber float64 `json:"active_shards_percent_as_number"` +} diff --git a/inputs/elasticsearch/collector/cluster_health_test.go b/inputs/elasticsearch/collector/cluster_health_test.go new file mode 100644 index 00000000..29f0a455 --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_health_test.go @@ -0,0 +1,349 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestClusterHealth(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/twitter + // curl http://localhost:9200/_cluster/health + + tests := []struct { + name string + file string + want string + }{ + { + name: "1.7.6", + file: "../fixtures/clusterhealth/1.7.6.json", + want: ` + # HELP elasticsearch_cluster_health_active_primary_shards The number of primary shards in your cluster. This is an aggregate total across all indices. + # TYPE elasticsearch_cluster_health_active_primary_shards gauge + elasticsearch_cluster_health_active_primary_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_active_shards Aggregate total of all shards across all indices, which includes replica shards. + # TYPE elasticsearch_cluster_health_active_shards gauge + elasticsearch_cluster_health_active_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_active_shards_percent Percentage of active shards in the cluster. + # TYPE elasticsearch_cluster_health_active_shards_percent gauge + elasticsearch_cluster_health_active_shards_percent{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_delayed_unassigned_shards Shards delayed to reduce reallocation overhead + # TYPE elasticsearch_cluster_health_delayed_unassigned_shards gauge + elasticsearch_cluster_health_delayed_unassigned_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_initializing_shards Count of shards that are being freshly created. + # TYPE elasticsearch_cluster_health_initializing_shards gauge + elasticsearch_cluster_health_initializing_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_number_of_data_nodes Number of data nodes in the cluster. + # TYPE elasticsearch_cluster_health_number_of_data_nodes gauge + elasticsearch_cluster_health_number_of_data_nodes{cluster="elasticsearch"} 1 + # HELP elasticsearch_cluster_health_number_of_in_flight_fetch The number of ongoing shard info requests. + # TYPE elasticsearch_cluster_health_number_of_in_flight_fetch gauge + elasticsearch_cluster_health_number_of_in_flight_fetch{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_number_of_nodes Number of nodes in the cluster. + # TYPE elasticsearch_cluster_health_number_of_nodes gauge + elasticsearch_cluster_health_number_of_nodes{cluster="elasticsearch"} 1 + # HELP elasticsearch_cluster_health_number_of_pending_tasks Cluster level changes which have not yet been executed + # TYPE elasticsearch_cluster_health_number_of_pending_tasks gauge + elasticsearch_cluster_health_number_of_pending_tasks{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_relocating_shards The number of shards that are currently moving from one node to another node. + # TYPE elasticsearch_cluster_health_relocating_shards gauge + elasticsearch_cluster_health_relocating_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_status Whether all primary and replica shards are allocated. + # TYPE elasticsearch_cluster_health_status gauge + elasticsearch_cluster_health_status{cluster="elasticsearch",color="green"} 0 + elasticsearch_cluster_health_status{cluster="elasticsearch",color="red"} 0 + elasticsearch_cluster_health_status{cluster="elasticsearch",color="yellow"} 1 + # HELP elasticsearch_cluster_health_task_max_waiting_in_queue_millis Tasks max time waiting in queue. + # TYPE elasticsearch_cluster_health_task_max_waiting_in_queue_millis gauge + elasticsearch_cluster_health_task_max_waiting_in_queue_millis{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_unassigned_shards The number of shards that exist in the cluster state, but cannot be found in the cluster itself. + # TYPE elasticsearch_cluster_health_unassigned_shards gauge + elasticsearch_cluster_health_unassigned_shards{cluster="elasticsearch"} 5 + `, + }, + { + name: "2.4.5", + file: "../fixtures/clusterhealth/2.4.5.json", + want: ` + # HELP elasticsearch_cluster_health_active_primary_shards The number of primary shards in your cluster. This is an aggregate total across all indices. + # TYPE elasticsearch_cluster_health_active_primary_shards gauge + elasticsearch_cluster_health_active_primary_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_active_shards Aggregate total of all shards across all indices, which includes replica shards. + # TYPE elasticsearch_cluster_health_active_shards gauge + elasticsearch_cluster_health_active_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_active_shards_percent Percentage of active shards in the cluster. + # TYPE elasticsearch_cluster_health_active_shards_percent gauge + elasticsearch_cluster_health_active_shards_percent{cluster="elasticsearch"} 50 + # HELP elasticsearch_cluster_health_delayed_unassigned_shards Shards delayed to reduce reallocation overhead + # TYPE elasticsearch_cluster_health_delayed_unassigned_shards gauge + elasticsearch_cluster_health_delayed_unassigned_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_initializing_shards Count of shards that are being freshly created. + # TYPE elasticsearch_cluster_health_initializing_shards gauge + elasticsearch_cluster_health_initializing_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_number_of_data_nodes Number of data nodes in the cluster. + # TYPE elasticsearch_cluster_health_number_of_data_nodes gauge + elasticsearch_cluster_health_number_of_data_nodes{cluster="elasticsearch"} 1 + # HELP elasticsearch_cluster_health_number_of_in_flight_fetch The number of ongoing shard info requests. + # TYPE elasticsearch_cluster_health_number_of_in_flight_fetch gauge + elasticsearch_cluster_health_number_of_in_flight_fetch{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_number_of_nodes Number of nodes in the cluster. + # TYPE elasticsearch_cluster_health_number_of_nodes gauge + elasticsearch_cluster_health_number_of_nodes{cluster="elasticsearch"} 1 + # HELP elasticsearch_cluster_health_number_of_pending_tasks Cluster level changes which have not yet been executed + # TYPE elasticsearch_cluster_health_number_of_pending_tasks gauge + elasticsearch_cluster_health_number_of_pending_tasks{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_relocating_shards The number of shards that are currently moving from one node to another node. + # TYPE elasticsearch_cluster_health_relocating_shards gauge + elasticsearch_cluster_health_relocating_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_status Whether all primary and replica shards are allocated. + # TYPE elasticsearch_cluster_health_status gauge + elasticsearch_cluster_health_status{cluster="elasticsearch",color="green"} 0 + elasticsearch_cluster_health_status{cluster="elasticsearch",color="red"} 0 + elasticsearch_cluster_health_status{cluster="elasticsearch",color="yellow"} 1 + # HELP elasticsearch_cluster_health_task_max_waiting_in_queue_millis Tasks max time waiting in queue. + # TYPE elasticsearch_cluster_health_task_max_waiting_in_queue_millis gauge + elasticsearch_cluster_health_task_max_waiting_in_queue_millis{cluster="elasticsearch"} 12 + # HELP elasticsearch_cluster_health_unassigned_shards The number of shards that exist in the cluster state, but cannot be found in the cluster itself. + # TYPE elasticsearch_cluster_health_unassigned_shards gauge + elasticsearch_cluster_health_unassigned_shards{cluster="elasticsearch"} 5 + `, + }, + { + name: "5.4.2", + file: "../fixtures/clusterhealth/5.4.2.json", + want: ` + # HELP elasticsearch_cluster_health_active_primary_shards The number of primary shards in your cluster. This is an aggregate total across all indices. + # TYPE elasticsearch_cluster_health_active_primary_shards gauge + elasticsearch_cluster_health_active_primary_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_active_shards Aggregate total of all shards across all indices, which includes replica shards. + # TYPE elasticsearch_cluster_health_active_shards gauge + elasticsearch_cluster_health_active_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_active_shards_percent Percentage of active shards in the cluster. + # TYPE elasticsearch_cluster_health_active_shards_percent gauge + elasticsearch_cluster_health_active_shards_percent{cluster="elasticsearch"} 50 + # HELP elasticsearch_cluster_health_delayed_unassigned_shards Shards delayed to reduce reallocation overhead + # TYPE elasticsearch_cluster_health_delayed_unassigned_shards gauge + elasticsearch_cluster_health_delayed_unassigned_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_initializing_shards Count of shards that are being freshly created. + # TYPE elasticsearch_cluster_health_initializing_shards gauge + elasticsearch_cluster_health_initializing_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_number_of_data_nodes Number of data nodes in the cluster. + # TYPE elasticsearch_cluster_health_number_of_data_nodes gauge + elasticsearch_cluster_health_number_of_data_nodes{cluster="elasticsearch"} 1 + # HELP elasticsearch_cluster_health_number_of_in_flight_fetch The number of ongoing shard info requests. + # TYPE elasticsearch_cluster_health_number_of_in_flight_fetch gauge + elasticsearch_cluster_health_number_of_in_flight_fetch{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_number_of_nodes Number of nodes in the cluster. + # TYPE elasticsearch_cluster_health_number_of_nodes gauge + elasticsearch_cluster_health_number_of_nodes{cluster="elasticsearch"} 1 + # HELP elasticsearch_cluster_health_number_of_pending_tasks Cluster level changes which have not yet been executed + # TYPE elasticsearch_cluster_health_number_of_pending_tasks gauge + elasticsearch_cluster_health_number_of_pending_tasks{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_relocating_shards The number of shards that are currently moving from one node to another node. + # TYPE elasticsearch_cluster_health_relocating_shards gauge + elasticsearch_cluster_health_relocating_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_status Whether all primary and replica shards are allocated. + # TYPE elasticsearch_cluster_health_status gauge + elasticsearch_cluster_health_status{cluster="elasticsearch",color="green"} 0 + elasticsearch_cluster_health_status{cluster="elasticsearch",color="red"} 0 + elasticsearch_cluster_health_status{cluster="elasticsearch",color="yellow"} 1 + # HELP elasticsearch_cluster_health_task_max_waiting_in_queue_millis Tasks max time waiting in queue. + # TYPE elasticsearch_cluster_health_task_max_waiting_in_queue_millis gauge + elasticsearch_cluster_health_task_max_waiting_in_queue_millis{cluster="elasticsearch"} 12 + # HELP elasticsearch_cluster_health_unassigned_shards The number of shards that exist in the cluster state, but cannot be found in the cluster itself. + # TYPE elasticsearch_cluster_health_unassigned_shards gauge + elasticsearch_cluster_health_unassigned_shards{cluster="elasticsearch"} 5 + `, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + c := NewClusterHealth(http.DefaultClient, u) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestClusterHealthIndices(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/twitter + // curl http://localhost:9200/_cluster/health + + tests := []struct { + name string + file string + want string + }{ + { + name: "1.7.6", + file: "../fixtures/clusterhealth/1.7.6.json", + want: ` + # HELP elasticsearch_cluster_health_indices_active_primary_shards The number of primary shards in your cluster. This is an aggregate total across all indices. + # TYPE elasticsearch_cluster_health_indices_active_primary_shards gauge + elasticsearch_cluster_health_indices_active_primary_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_indices_active_shards Aggregate total of all shards across all indices, which includes replica shards. + # TYPE elasticsearch_cluster_health_indices_active_shards gauge + elasticsearch_cluster_health_indices_active_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_indices_initializing_shards Count of shards that are being freshly created. + # TYPE elasticsearch_cluster_health_indices_initializing_shards gauge + elasticsearch_cluster_health_indices_initializing_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_number_of_replicas Number of replicas in the cluster. + # TYPE elasticsearch_cluster_health_indices_number_of_replicas gauge + elasticsearch_cluster_health_indices_number_of_replicas{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_number_of_shards Number of shards in the cluster. + # TYPE elasticsearch_cluster_health_indices_number_of_shards gauge + elasticsearch_cluster_health_indices_number_of_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_relocating_shards The number of shards that are currently moving from one node to another node. + # TYPE elasticsearch_cluster_health_indices_relocating_shards gauge + elasticsearch_cluster_health_indices_relocating_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_status Whether all primary and replica shards are allocated. + # TYPE elasticsearch_cluster_health_indices_status gauge + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="green"} 0 + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="red"} 0 + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="yellow"} 1 + # HELP elasticsearch_cluster_health_indices_unassigned_shards The number of shards that exist in the cluster state, but cannot be found in the cluster itself. + # TYPE elasticsearch_cluster_health_indices_unassigned_shards gauge + elasticsearch_cluster_health_indices_unassigned_shards{cluster="elasticsearch"} 5 + `, + }, + { + name: "2.4.5", + file: "../fixtures/clusterhealth/2.4.5.json", + want: ` + # HELP elasticsearch_cluster_health_indices_active_primary_shards The number of primary shards in your cluster. This is an aggregate total across all indices. + # TYPE elasticsearch_cluster_health_indices_active_primary_shards gauge + elasticsearch_cluster_health_indices_active_primary_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_indices_active_shards Aggregate total of all shards across all indices, which includes replica shards. + # TYPE elasticsearch_cluster_health_indices_active_shards gauge + elasticsearch_cluster_health_indices_active_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_indices_initializing_shards Count of shards that are being freshly created. + # TYPE elasticsearch_cluster_health_indices_initializing_shards gauge + elasticsearch_cluster_health_indices_initializing_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_number_of_replicas Number of replicas in the cluster. + # TYPE elasticsearch_cluster_health_indices_number_of_replicas gauge + elasticsearch_cluster_health_indices_number_of_replicas{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_number_of_shards Number of shards in the cluster. + # TYPE elasticsearch_cluster_health_indices_number_of_shards gauge + elasticsearch_cluster_health_indices_number_of_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_relocating_shards The number of shards that are currently moving from one node to another node. + # TYPE elasticsearch_cluster_health_indices_relocating_shards gauge + elasticsearch_cluster_health_indices_relocating_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_status Whether all primary and replica shards are allocated. + # TYPE elasticsearch_cluster_health_indices_status gauge + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="green"} 0 + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="red"} 0 + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="yellow"} 1 + # HELP elasticsearch_cluster_health_indices_unassigned_shards The number of shards that exist in the cluster state, but cannot be found in the cluster itself. + # TYPE elasticsearch_cluster_health_indices_unassigned_shards gauge + elasticsearch_cluster_health_indices_unassigned_shards{cluster="elasticsearch"} 5 + `, + }, + { + name: "5.4.2", + file: "../fixtures/clusterhealth/5.4.2.json", + want: ` + # HELP elasticsearch_cluster_health_indices_active_primary_shards The number of primary shards in your cluster. This is an aggregate total across all indices. + # TYPE elasticsearch_cluster_health_indices_active_primary_shards gauge + elasticsearch_cluster_health_indices_active_primary_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_indices_active_shards Aggregate total of all shards across all indices, which includes replica shards. + # TYPE elasticsearch_cluster_health_indices_active_shards gauge + elasticsearch_cluster_health_indices_active_shards{cluster="elasticsearch"} 5 + # HELP elasticsearch_cluster_health_indices_initializing_shards Count of shards that are being freshly created. + # TYPE elasticsearch_cluster_health_indices_initializing_shards gauge + elasticsearch_cluster_health_indices_initializing_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_number_of_replicas Number of replicas in the cluster. + # TYPE elasticsearch_cluster_health_indices_number_of_replicas gauge + elasticsearch_cluster_health_indices_number_of_replicas{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_number_of_shards Number of shards in the cluster. + # TYPE elasticsearch_cluster_health_indices_number_of_shards gauge + elasticsearch_cluster_health_indices_number_of_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_relocating_shards The number of shards that are currently moving from one node to another node. + # TYPE elasticsearch_cluster_health_indices_relocating_shards gauge + elasticsearch_cluster_health_indices_relocating_shards{cluster="elasticsearch"} 0 + # HELP elasticsearch_cluster_health_indices_status Whether all primary and replica shards are allocated. + # TYPE elasticsearch_cluster_health_indices_status gauge + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="green"} 0 + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="red"} 0 + elasticsearch_cluster_health_indices_status{cluster="elasticsearch",color="yellow"} 1 + # HELP elasticsearch_cluster_health_indices_unassigned_shards The number of shards that exist in the cluster state, but cannot be found in the cluster itself. + # TYPE elasticsearch_cluster_health_indices_unassigned_shards gauge + elasticsearch_cluster_health_indices_unassigned_shards{cluster="elasticsearch"} 5 + `, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + c := NewClusterHealthIndices(http.DefaultClient, u) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/cluster_info.go b/inputs/elasticsearch/collector/cluster_info.go new file mode 100644 index 00000000..0f1aa436 --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_info.go @@ -0,0 +1,106 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + + "github.com/blang/semver/v4" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + registerCollector("cluster-info", defaultEnabled, NewClusterInfo) +} + +type ClusterInfoCollector struct { + u *url.URL + hc *http.Client +} + +func NewClusterInfo(u *url.URL, hc *http.Client) (Collector, error) { + return &ClusterInfoCollector{ + u: u, + hc: hc, + }, nil +} + +var clusterInfoDesc = map[string]*prometheus.Desc{ + "version": prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "version"), + "Elasticsearch version information.", + []string{ + "cluster", + "cluster_uuid", + "build_date", + "build_hash", + "version", + "lucene_version", + }, + nil, + ), +} + +// ClusterInfoResponse is the cluster info retrievable from the / endpoint +type ClusterInfoResponse struct { + Name string `json:"name"` + ClusterName string `json:"cluster_name"` + ClusterUUID string `json:"cluster_uuid"` + Version VersionInfo `json:"version"` + Tagline string `json:"tagline"` +} + +// VersionInfo is the version info retrievable from the / endpoint, embedded in ClusterInfoResponse +type VersionInfo struct { + Number semver.Version `json:"number"` + BuildHash string `json:"build_hash"` + BuildDate string `json:"build_date"` + BuildSnapshot bool `json:"build_snapshot"` + LuceneVersion semver.Version `json:"lucene_version"` +} + +func (c *ClusterInfoCollector) Update(_ context.Context, ch chan<- prometheus.Metric) error { + resp, err := c.hc.Get(c.u.String()) + if err != nil { + return err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + var info ClusterInfoResponse + err = json.Unmarshal(b, &info) + if err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + clusterInfoDesc["version"], + prometheus.GaugeValue, + 1, + info.ClusterName, + info.ClusterUUID, + info.Version.BuildDate, + info.Version.BuildHash, + info.Version.Number.String(), + info.Version.LuceneVersion.String(), + ) + + return nil +} diff --git a/inputs/elasticsearch/collector/cluster_info_test.go b/inputs/elasticsearch/collector/cluster_info_test.go new file mode 100644 index 00000000..7060cf5a --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_info_test.go @@ -0,0 +1,92 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestClusterInfo(t *testing.T) { + // Testcases created using: + // docker run -p 9200:9200 -e "discovery.type=single-node" elasticsearch:${VERSION} + // curl http://localhost:9200/ > fixtures/cluster_info/${VERSION}.json + + tests := []struct { + name string + file string + want string + }{ + { + name: "2.4.5", + file: "../fixtures/clusterinfo/2.4.5.json", + want: `# HELP elasticsearch_version Elasticsearch version information. + # TYPE elasticsearch_version gauge + elasticsearch_version{build_date="",build_hash="c849dd13904f53e63e88efc33b2ceeda0b6a1276",cluster="elasticsearch",cluster_uuid="3qps7bcWTqyzV49ApmPVfw",lucene_version="5.5.4",version="2.4.5"} 1 + `, + }, + { + name: "5.4.2", + file: "../fixtures/clusterinfo/5.4.2.json", + want: `# HELP elasticsearch_version Elasticsearch version information. + # TYPE elasticsearch_version gauge + elasticsearch_version{build_date="2017-06-15T02:29:28.122Z",build_hash="929b078",cluster="elasticsearch",cluster_uuid="kbqi7yhQT-WlPdGL2m0xJg",lucene_version="6.5.1",version="5.4.2"} 1 + `, + }, + { + name: "7.13.1", + file: "../fixtures/clusterinfo/7.13.1.json", + want: `# HELP elasticsearch_version Elasticsearch version information. + # TYPE elasticsearch_version gauge + elasticsearch_version{build_date="2021-05-28T17:40:59.346932922Z",build_hash="9a7758028e4ea59bcab41c12004603c5a7dd84a9",cluster="docker-cluster",cluster_uuid="aCMrCY1VQpqJ6U4Sw_xdiw",lucene_version="8.8.2",version="7.13.1"} 1 + `, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + c, err := NewClusterInfo(u, http.DefaultClient) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(wrapCollector{c}, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/cluster_settings.go b/inputs/elasticsearch/collector/cluster_settings.go new file mode 100644 index 00000000..3d762134 --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_settings.go @@ -0,0 +1,174 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strconv" + + "github.com/imdario/mergo" + "github.com/prometheus/client_golang/prometheus" +) + +// ClusterSettings information struct +type ClusterSettings struct { + client *http.Client + url *url.URL + + up prometheus.Gauge + shardAllocationEnabled prometheus.Gauge + maxShardsPerNode prometheus.Gauge + totalScrapes, jsonParseFailures prometheus.Counter +} + +// NewClusterSettings defines Cluster Settings Prometheus metrics +func NewClusterSettings(client *http.Client, url *url.URL) *ClusterSettings { + return &ClusterSettings{ + client: client, + url: url, + + up: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "up"), + Help: "Was the last scrape of the Elasticsearch cluster settings endpoint successful.", + }), + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "total_scrapes"), + Help: "Current total Elasticsearch cluster settings scrapes.", + }), + shardAllocationEnabled: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "shard_allocation_enabled"), + Help: "Current mode of cluster wide shard routing allocation settings.", + }), + maxShardsPerNode: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "max_shards_per_node"), + Help: "Current maximum number of shards per node setting.", + }), + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + } +} + +// Describe add Snapshots metrics descriptions +func (cs *ClusterSettings) Describe(ch chan<- *prometheus.Desc) { + ch <- cs.up.Desc() + ch <- cs.totalScrapes.Desc() + ch <- cs.shardAllocationEnabled.Desc() + ch <- cs.maxShardsPerNode.Desc() + ch <- cs.jsonParseFailures.Desc() +} + +func (cs *ClusterSettings) getAndParseURL(u *url.URL, data interface{}) error { + res, err := cs.client.Get(u.String()) + if err != nil { + return fmt.Errorf("failed to get from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + cs.jsonParseFailures.Inc() + return err + } + + if err := json.Unmarshal(bts, data); err != nil { + cs.jsonParseFailures.Inc() + return err + } + + return nil +} + +func (cs *ClusterSettings) fetchAndDecodeClusterSettingsStats() (ClusterSettingsResponse, error) { + + u := *cs.url + u.Path = path.Join(u.Path, "/_cluster/settings") + q := u.Query() + q.Set("include_defaults", "true") + u.RawQuery = q.Encode() + u.RawPath = q.Encode() + var csfr ClusterSettingsFullResponse + var csr ClusterSettingsResponse + err := cs.getAndParseURL(&u, &csfr) + if err != nil { + return csr, err + } + err = mergo.Merge(&csr, csfr.Defaults, mergo.WithOverride) + if err != nil { + return csr, err + } + err = mergo.Merge(&csr, csfr.Persistent, mergo.WithOverride) + if err != nil { + return csr, err + } + err = mergo.Merge(&csr, csfr.Transient, mergo.WithOverride) + + return csr, err +} + +// Collect gets cluster settings metric values +func (cs *ClusterSettings) Collect(ch chan<- prometheus.Metric) { + + cs.totalScrapes.Inc() + defer func() { + ch <- cs.up + ch <- cs.totalScrapes + ch <- cs.jsonParseFailures + ch <- cs.shardAllocationEnabled + ch <- cs.maxShardsPerNode + }() + + csr, err := cs.fetchAndDecodeClusterSettingsStats() + if err != nil { + cs.shardAllocationEnabled.Set(0) + cs.up.Set(0) + log.Println("failed to fetch and decode cluster settings stats, err: ", err) + return + } + cs.up.Set(1) + + shardAllocationMap := map[string]int{ + "all": 0, + "primaries": 1, + "new_primaries": 2, + "none": 3, + } + + cs.shardAllocationEnabled.Set(float64(shardAllocationMap[csr.Cluster.Routing.Allocation.Enabled])) + + if maxShardsPerNodeString, ok := csr.Cluster.MaxShardsPerNode.(string); ok { + maxShardsPerNode, err := strconv.ParseInt(maxShardsPerNodeString, 10, 64) + if err == nil { + cs.maxShardsPerNode.Set(float64(maxShardsPerNode)) + } + } +} diff --git a/inputs/elasticsearch/collector/cluster_settings_reponse.go b/inputs/elasticsearch/collector/cluster_settings_reponse.go new file mode 100644 index 00000000..ac2fcb8d --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_settings_reponse.go @@ -0,0 +1,43 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +// ClusterSettingsFullResponse is a representation of a Elasticsearch Cluster Settings +type ClusterSettingsFullResponse struct { + Defaults ClusterSettingsResponse `json:"defaults"` + Persistent ClusterSettingsResponse `json:"persistent"` + Transient ClusterSettingsResponse `json:"transient"` +} + +// ClusterSettingsResponse is a representation of a Elasticsearch Cluster Settings +type ClusterSettingsResponse struct { + Cluster Cluster `json:"cluster"` +} + +// Cluster is a representation of a Elasticsearch Cluster Settings +type Cluster struct { + Routing Routing `json:"routing"` + // This can be either a JSON object (which does not contain the value we are interested in) or a string + MaxShardsPerNode interface{} `json:"max_shards_per_node"` +} + +// Routing is a representation of a Elasticsearch Cluster shard routing configuration +type Routing struct { + Allocation Allocation `json:"allocation"` +} + +// Allocation is a representation of a Elasticsearch Cluster shard routing allocation settings +type Allocation struct { + Enabled string `json:"enable"` +} diff --git a/inputs/elasticsearch/collector/cluster_settings_test.go b/inputs/elasticsearch/collector/cluster_settings_test.go new file mode 100644 index 00000000..e24c7989 --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_settings_test.go @@ -0,0 +1,91 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" +) + +func TestClusterSettingsStats(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl http://localhost:9200/_cluster/settings/?include_defaults=true + files := []string{"../fixtures/settings-5.4.2.json", "../fixtures/settings-merge-5.4.2.json"} + for _, filename := range files { + f, _ := os.Open(filename) + defer f.Close() + for hn, handler := range map[string]http.Handler{ + "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + }), + } { + ts := httptest.NewServer(handler) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + c := NewClusterSettings(http.DefaultClient, u) + nsr, err := c.fetchAndDecodeClusterSettingsStats() + if err != nil { + t.Fatalf("Failed to fetch or decode cluster settings stats: %s", err) + } + t.Logf("[%s/%s] Cluster Settings Stats Response: %+v", hn, filename, nsr) + if nsr.Cluster.Routing.Allocation.Enabled != "ALL" { + t.Errorf("Wrong setting for cluster routing allocation enabled") + } + if nsr.Cluster.MaxShardsPerNode != nil { + t.Errorf("MaxShardsPerNode should be empty on older releases") + } + } + } +} + +func TestClusterMaxShardsPerNode(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl http://localhost:9200/_cluster/settings/?include_defaults=true + files := []string{"../fixtures/settings-7.3.0.json"} + for _, filename := range files { + f, _ := os.Open(filename) + defer f.Close() + for hn, handler := range map[string]http.Handler{ + "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + }), + } { + ts := httptest.NewServer(handler) + defer ts.Close() + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + c := NewClusterSettings(http.DefaultClient, u) + nsr, err := c.fetchAndDecodeClusterSettingsStats() + if err != nil { + t.Fatalf("Failed to fetch or decode cluster settings stats: %s", err) + } + t.Logf("[%s/%s] Cluster Settings Stats Response: %+v", hn, filename, nsr) + if nsr.Cluster.MaxShardsPerNode != "1000" { + t.Errorf("Wrong value for MaxShardsPerNode") + } + } + } +} diff --git a/inputs/elasticsearch/collector/cluster_stats.go b/inputs/elasticsearch/collector/cluster_stats.go new file mode 100644 index 00000000..bd7892fc --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_stats.go @@ -0,0 +1,889 @@ +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +type clusterStatsMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(clusterHealth ClusterStatsResponse) float64 + Labels func(cluster ClusterStatsResponse) []string +} + +// ClusterStats type defines the collector struct +type ClusterStats struct { + client *http.Client + url *url.URL + + metrics []*clusterStatsMetric +} + +var ( + defaultClusterStatsLabels = []string{"cluster", "node", "status"} + defaultClusterStatsLabelValues = func(cluster ClusterStatsResponse) []string { + return []string{ + cluster.ClusterName, + cluster.NodeName, + cluster.Status, + } + } +) + +// NewClusterStats defines Nodes Prometheus metrics +func NewClusterStats(client *http.Client, url *url.URL) *ClusterStats { + return &ClusterStats{ + client: client, + url: url, + + metrics: []*clusterStatsMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "count"), + "Completion in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Count) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "completion_size_in_bytes"), + "Completion in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Completion.Size) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "docs_count"), + "Count of documents on this cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Docs.Count) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "docs_deleted"), + "Count of deleted documents on this cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Docs.Deleted) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "fielddata_evictions"), + "Evictions from field data", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.FieldData.Evictions) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "fielddata_memory_size_in_bytes"), + "Field data cache memory usage in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.FieldData.MemorySize) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "query_cache_cache_count"), + "Query cache cache count", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.QueryCache.CacheCount) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "query_cache_cache_size"), + "Query cache cache size", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.QueryCache.CacheSize) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "query_cache_evictions"), + "Evictions from query cache", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.QueryCache.Evictions) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "query_cache_hit_count"), + "Query cache count", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.QueryCache.HitCount) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "query_cache_memory_size_in_bytes"), + "Query cache memory usage in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.QueryCache.MemorySize) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "query_cache_miss_count"), + "Query miss count", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.QueryCache.MissCount) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "query_cache_total_count"), + "Query cache total count", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.QueryCache.TotalCount) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_count"), + "Count of index segments on this cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.Count) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_doc_values_memory_in_bytes"), + "Count of doc values memory", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.DocValuesMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_fixed_bit_set_memory_in_bytes"), + "Count of fixed bit set", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.FixedBitSet) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_index_writer_memory_in_bytes"), + "Count of memory for index writer on this cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.IndexWriterMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_max_unsafe_auto_id_timestamp"), + "Count of memory for index writer on this cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.MaxUnsafeAutoIDTimestamp) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_memory_in_bytes"), + "Current memory size of segments in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.Memory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_norms_memory_in_bytes"), + "Count of memory used by norms", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.NormsMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_points_memory_in_bytes"), + "Point values memory usage in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.PointsMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_stored_fields_memory_in_bytes"), + "Count of stored fields memory", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.StoredFieldsMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_term_vectors_memory_in_bytes"), + "Term vectors memory usage in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.TermVectorsMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_terms_memory_in_bytes"), + "Count of terms in memory for this cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.TermsMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "segments_version_map_memory_in_bytes"), + "Version map memory usage in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Segments.VersionMapMemory) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_total"), + "Total number of shards in the cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Total + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_replication"), + "Number of shards replication", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Replication + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_primaries"), + "Number of primary shards in the cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Primaries + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_primaries_avg"), + "Average number of primary shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Primaries.Avg + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_primaries_max"), + "Max number of primary shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Primaries.Max + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_primaries_min"), + "Min number of primary shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Primaries.Min + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_replication_avg"), + "Average number of replication shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Replicas.Avg + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_replication_max"), + "Max number of replication shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Replicas.Max + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_replication_min"), + "Min number of replication shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Replicas.Min + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_shards_avg"), + "Average number of shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Shards.Avg + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_shards_max"), + "Max number of shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Shards.Max + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "shards_index_shards_min"), + "Min number of shards per index", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Indices.Shards.Index.Shards.Min + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "store_size_in_bytes"), + "Current size of the store in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Store.Size) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "total_data_set_size_in_bytes"), + "Total data set size in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Store.TotalDataSetSize) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_indices", "reserved_in_bytes"), + "Reserved size in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Indices.Store.Reserved) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "count_coordinating_only"), + "Count of coordinating only nodes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.Count.CoordinatingOnly) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "count_data"), + "Count of data nodes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.Count.Data) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "count_ingest"), + "Count of ingest nodes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.Count.Ingest) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "count_master"), + "Count of master nodes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.Count.Master) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "count_total"), + "Total count of nodes in the cluster", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.Count.Total) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "fs_available_in_bytes"), + "Available disk space in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.FS.AvailableInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "fs_free_in_bytes"), + "Free disk space in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.FS.FreeInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "fs_total_in_bytes"), + "Total disk space in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.FS.TotalInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "jvm_max_uptime_seconds"), + "Max uptime in milliseconds", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.JVM.MaxUptimeInMillis) / 1000 + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "jvm_mem_heap_max_in_bytes"), + "Max heap memory in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.JVM.Mem.HeapMaxInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "jvm_mem_heap_used_in_bytes"), + "Used heap memory in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.JVM.Mem.HeapUsedInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "jvm_threads"), + "Number of threads", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.JVM.Threads) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "network_types_http_types_security4"), + "HTTP security4 network types", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.NetWorkTypes.HTTPTypes.Security) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "network_types_transport_types_security4"), + "Transport security4 network types", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.NetWorkTypes.TransportTypes.Security) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "os_allocated_processors"), + "Allocated processors", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.OS.AllocatedProcessors) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "os_available_processors"), + "Available processors", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.OS.AvailableProcessors) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "os_mem_free_in_bytes"), + "Free memory in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.OS.Mem.FreeInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "os_mem_free_percent"), + "Free memory in percent", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.OS.Mem.FreePercent) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "os_mem_total_in_bytes"), + "Total memory in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.OS.Mem.TotalInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "os_mem_used_in_bytes"), + "Used memory in bytes", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.OS.Mem.UsedInBytes) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "os_mem_used_percent"), + "Used memory in percent", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.OS.Mem.UsedPercent) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "process_cpu_percent"), + "Process CPU in percent", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return float64(cluster.Nodes.Process.CPU.Percent) + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "process_open_file_descriptors_avg"), + "Average number of open file descriptors", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Nodes.Process.OpenFileDescriptors.Avg + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "process_open_file_descriptors_max"), + "Max number of open file descriptors", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Nodes.Process.OpenFileDescriptors.Max + }, + Labels: defaultClusterStatsLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "clusterstats_nodes", "process_open_file_descriptors_min"), + "Min number of open file descriptors", + defaultClusterStatsLabels, nil, + ), + Value: func(cluster ClusterStatsResponse) float64 { + return cluster.Nodes.Process.OpenFileDescriptors.Min + }, + Labels: defaultClusterStatsLabelValues, + }, + }, + } +} + +// Describe add metrics descriptions +func (c *ClusterStats) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range c.metrics { + ch <- metric.Desc + } +} + +func (c *ClusterStats) fetchAndDecodeClusterStats() (ClusterStatsResponse, error) { + var chr ClusterStatsResponse + + u := *c.url + u.Path = path.Join(u.Path, "/_cluster/stats") + res, err := c.client.Get(u.String()) + if err != nil { + return chr, fmt.Errorf("failed to get cluster stats from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return chr, err + } + + if err := json.Unmarshal(bts, &chr); err != nil { + return chr, err + } + + return chr, nil +} + +// Collect gets clusters metric values +func (c *ClusterStats) Collect(ch chan<- prometheus.Metric) { + clusterStatsResp, err := c.fetchAndDecodeClusterStats() + if err != nil { + log.Println("failed to fetch and decode cluster health, err: ", err) + return + } + + for _, metric := range c.metrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(clusterStatsResp), + defaultClusterStatsLabelValues(clusterStatsResp)..., + ) + } +} diff --git a/inputs/elasticsearch/collector/cluster_stats_response.go b/inputs/elasticsearch/collector/cluster_stats_response.go new file mode 100644 index 00000000..985f090f --- /dev/null +++ b/inputs/elasticsearch/collector/cluster_stats_response.go @@ -0,0 +1,208 @@ +package collector + +// ClusterStatsResponse defines node stats information structure for nodes +type ClusterStatsResponse struct { + NodeName string `json:"node_name"` + ClusterName string `json:"cluster_name"` + Status string `json:"status"` + Indices ClusterStatsIndicesResponse `json:"indices"` + Nodes ClusterStatsNodesResponse `json:"nodes"` +} + +// ClusterStatsIndicesResponse is a representation of a indices stats (size, document count, indexing and deletion times, search times, field cache size, merges and flushes) +type ClusterStatsIndicesResponse struct { + Count int64 `json:"count"` + Docs ClusterStatsIndicesDocsResponse + Indexing ClusterStatsIndicesIndexingResponse + FieldData ClusterStatsIndicesCacheResponse `json:"fielddata"` + FilterCache ClusterStatsIndicesCacheResponse `json:"filter_cache"` + QueryCache ClusterStatsIndicesCacheResponse `json:"query_cache"` + RequestCache ClusterStatsIndicesCacheResponse `json:"request_cache"` + Segments ClusterStatsIndicesSegmentsResponse + Completion ClusterStatsIndicesCompletionResponse + Shards ClusterStatsIndicesShardsResponse + Store ClusterStatsIndicesStoreResponse +} + +// ClusterStatsIndicesDocsResponse defines node stats docs information structure for indices +type ClusterStatsIndicesDocsResponse struct { + Count int64 `json:"count"` + Deleted int64 `json:"deleted"` +} + +// ClusterStatsIndicesIndexingResponse defines node stats indexing information structure for indices +type ClusterStatsIndicesIndexingResponse struct { + IndexTotal int64 `json:"index_total"` + IndexTime int64 `json:"index_time_in_millis"` + IndexCurrent int64 `json:"index_current"` + DeleteTotal int64 `json:"delete_total"` + DeleteTime int64 `json:"delete_time_in_millis"` + DeleteCurrent int64 `json:"delete_current"` + IsThrottled bool `json:"is_throttled"` + ThrottleTime int64 `json:"throttle_time_in_millis"` +} + +// ClusterStatsIndicesCacheResponse defines node stats cache information structure for indices +type ClusterStatsIndicesCacheResponse struct { + Evictions int64 `json:"evictions"` + MemorySize int64 `json:"memory_size_in_bytes"` + CacheCount int64 `json:"cache_count"` + CacheSize int64 `json:"cache_size"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` + TotalCount int64 `json:"total_count"` +} + +// ClusterStatsIndicesSegmentsResponse defines node stats segments information structure for indices +type ClusterStatsIndicesSegmentsResponse struct { + Count int64 `json:"count"` + Memory int64 `json:"memory_in_bytes"` + TermsMemory int64 `json:"terms_memory_in_bytes"` + IndexWriterMemory int64 `json:"index_writer_memory_in_bytes"` + NormsMemory int64 `json:"norms_memory_in_bytes"` + StoredFieldsMemory int64 `json:"stored_fields_memory_in_bytes"` + FixedBitSet int64 `json:"fixed_bit_set_memory_in_bytes"` + DocValuesMemory int64 `json:"doc_values_memory_in_bytes"` + TermVectorsMemory int64 `json:"term_vectors_memory_in_bytes"` + PointsMemory int64 `json:"points_memory_in_bytes"` + VersionMapMemory int64 `json:"version_map_memory_in_bytes"` + MaxUnsafeAutoIDTimestamp int64 `json:"max_unsafe_auto_id_timestamp"` +} + +// ClusterStatsIndicesCompletionResponse defines node stats completion information structure for indices +type ClusterStatsIndicesCompletionResponse struct { + Size int64 `json:"size_in_bytes"` +} + +// ClusterStatsIndicesShardsResponse defines node stats shards information structure for indices +type ClusterStatsIndicesShardsResponse struct { + Total float64 `json:"total"` + Primaries float64 `json:"primaries"` + Replication float64 `json:"replication"` + Index ClusterStatsIndicesShardsIndexResponse `json:"index"` +} + +// ClusterStatsIndicesShardsIndexResponse defines node stats shards index information structure for indices +type ClusterStatsIndicesShardsIndexResponse struct { + Shards ClusterStatsValueResponse `json:"shards"` + Primaries ClusterStatsValueResponse `json:"primaries"` + Replicas ClusterStatsValueResponse `json:"replicas"` +} + +// ClusterStatsValueResponse defines node stats shards index value information structure for indices +type ClusterStatsValueResponse struct { + Avg float64 `json:"avg"` + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +// ClusterStatsIndicesStoreResponse defines node stats store information structure for indices +type ClusterStatsIndicesStoreResponse struct { + Size int64 `json:"size_in_bytes"` + TotalDataSetSize int64 `json:"total_data_set_size_in_bytes"` + Reserved int64 `json:"reserved_in_bytes"` +} + +// ClusterStatsNodesResponse defines node stats information structure for nodes +type ClusterStatsNodesResponse struct { + Count ClusterStatsNodesCountResponse `json:"count"` + JVM ClusterStatsNodesJVMResponse `json:"jvm"` + FS ClusterStatsNodesFSResponse `json:"fs"` + OS ClusterStatsNodesOSResponse `json:"os"` + Process ClusterStatsNodesProcessResponse `json:"process"` + NetWorkTypes ClusterStatsNodesNetworkTypesResponse `json:"network_types"` + Versions []string `json:"versions"` +} + +// ClusterStatsNodesCountResponse defines node stats count information structure for nodes +type ClusterStatsNodesCountResponse struct { + Total int64 `json:"total"` + CoordinatingOnly int64 `json:"coordinating_only"` + Data int64 `json:"data"` + DataCold int64 `json:"data_cold"` + DataContent int64 `json:"data_content"` + DataFrozen int64 `json:"data_frozen"` + DataHot int64 `json:"data_hot"` + DataWarm int64 `json:"data_warm"` + Ingest int64 `json:"ingest"` + Master int64 `json:"master"` + Ml int64 `json:"ml"` + RemoteClusterClient int64 `json:"remote_cluster_client"` + Transform int64 `json:"transform"` + VotingOnly int64 `json:"voting_only"` +} + +// ClusterStatsNodesJVMResponse defines node stats JVM information structure for nodes +type ClusterStatsNodesJVMResponse struct { + MaxUptimeInMillis int64 `json:"max_uptime_in_millis"` + Mem ClusterStatsNodesJVMMemResponse `json:"mem"` + Versions []ClusterStatsNodesJVMVersionsResponse `json:"versions"` + Threads int64 `json:"threads"` +} + +// ClusterStatsNodesJVMMemResponse defines node stats JVM memory information structure for nodes +type ClusterStatsNodesJVMMemResponse struct { + HeapUsedInBytes int64 `json:"heap_used_in_bytes"` + HeapMaxInBytes int64 `json:"heap_max_in_bytes"` +} + +// ClusterStatsNodesJVMVersionsResponse defines node stats JVM versions information structure for nodes +type ClusterStatsNodesJVMVersionsResponse struct { + Version string `json:"version"` + VMName string `json:"vm_name"` + VMVersion string `json:"vm_version"` + VMVendor string `json:"vm_vendor"` + Count int64 `json:"count"` +} + +// ClusterStatsNodesFSResponse defines node stats FS information structure for nodes +type ClusterStatsNodesFSResponse struct { + TotalInBytes int64 `json:"total_in_bytes"` + FreeInBytes int64 `json:"free_in_bytes"` + AvailableInBytes int64 `json:"available_in_bytes"` +} + +// ClusterStatsNodesOSResponse defines node stats OS information structure for nodes +type ClusterStatsNodesOSResponse struct { + AvailableProcessors int64 `json:"available_processors"` + AllocatedProcessors int64 `json:"allocated_processors"` + Mem ClusterStatsNodesOSMemResponse `json:"mem"` + Names []ClusterStatsNodesOSNamesResponse `json:"names"` +} + +// ClusterStatsNodesOSMemResponse defines node stats OS memory information structure for nodes +type ClusterStatsNodesOSMemResponse struct { + TotalInBytes int64 `json:"total_in_bytes"` + FreeInBytes int64 `json:"free_in_bytes"` + UsedInBytes int64 `json:"used_in_bytes"` + FreePercent int64 `json:"free_percent"` + UsedPercent int64 `json:"used_percent"` +} + +// ClusterStatsNodesOSNamesResponse defines node stats OS names information structure for nodes +type ClusterStatsNodesOSNamesResponse struct { + Name string `json:"name"` + Count int64 `json:"count"` +} + +// ClusterStatsNodesProcessResponse defines node stats process information structure for nodes +type ClusterStatsNodesProcessResponse struct { + OpenFileDescriptors ClusterStatsValueResponse `json:"open_file_descriptors"` + CPU ClusterStatsNodesProcessCPUResponse `json:"cpu"` +} + +// ClusterStatsNodesProcessCPUResponse defines node stats process CPU information structure for nodes +type ClusterStatsNodesProcessCPUResponse struct { + Percent int64 `json:"percent"` +} + +// ClusterStatsNodesNetworkTypesResponse defines node stats network types information structure for nodes +type ClusterStatsNodesNetworkTypesResponse struct { + TransportTypes ClusterStatsNodesNetworkTypeResponse `json:"transport_types"` + HTTPTypes ClusterStatsNodesNetworkTypeResponse `json:"http_types"` +} + +// ClusterStatsNodesNetworkTypeResponse defines node stats network type information structure for nodes +type ClusterStatsNodesNetworkTypeResponse struct { + Security int64 `json:"security4"` +} diff --git a/inputs/elasticsearch/collector/collector.go b/inputs/elasticsearch/collector/collector.go new file mode 100644 index 00000000..671c5687 --- /dev/null +++ b/inputs/elasticsearch/collector/collector.go @@ -0,0 +1,215 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package collector includes all individual collectors to gather and export elasticsearch metrics. +package collector + +import ( + "context" + "errors" + "fmt" + "github.com/alecthomas/kingpin/v2" + "log" + "net/http" + "net/url" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + // Namespace defines the common namespace to be used by all metrics. + namespace = "elasticsearch" + + defaultEnabled = true + defaultDisabled = false +) + +type factoryFunc func(u *url.URL, hc *http.Client) (Collector, error) + +var ( + factories = make(map[string]factoryFunc) + initiatedCollectorsMtx = sync.Mutex{} + initiatedCollectors = make(map[string]Collector) + collectorState = make(map[string]*bool) + forcedCollectors = map[string]bool{} // collectors which have been explicitly enabled or disabled +) + +var ( + scrapeDurationDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "scrape", "duration_seconds"), + "elasticsearch_exporter: Duration of a collector scrape.", + []string{"collector"}, + nil, + ) + scrapeSuccessDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "scrape", "success"), + "elasticsearch_exporter: Whether a collector succeeded.", + []string{"collector"}, + nil, + ) +) + +// Collector is the interface a collector has to implement. +type Collector interface { + // Get new metrics and expose them via prometheus registry. + Update(context.Context, chan<- prometheus.Metric) error +} + +func registerCollector(name string, isDefaultEnabled bool, createFunc factoryFunc) { + var helpDefaultState string + if isDefaultEnabled { + helpDefaultState = "enabled" + } else { + helpDefaultState = "disabled" + } + + // Create flag for this collector + flagName := fmt.Sprintf("collector.%s", name) + flagHelp := fmt.Sprintf("Enable the %s collector (default: %s).", name, helpDefaultState) + defaultValue := fmt.Sprintf("%v", isDefaultEnabled) + + flag := kingpin.Flag(flagName, flagHelp).Default(defaultValue).Action(collectorFlagAction(name)).Bool() + collectorState[name] = flag + + // Register the create function for this collector + factories[name] = createFunc +} + +type ElasticsearchCollector struct { + Collectors map[string]Collector + esURL *url.URL + httpClient *http.Client +} + +type Option func(*ElasticsearchCollector) error + +// NewElasticsearchCollector creates a new ElasticsearchCollector +func NewElasticsearchCollector(filters []string, options ...Option) (*ElasticsearchCollector, error) { + e := &ElasticsearchCollector{} + // Apply options to customize the collector + for _, o := range options { + if err := o(e); err != nil { + return nil, err + } + } + + f := make(map[string]bool) + for _, filter := range filters { + enabled, exist := collectorState[filter] + if !exist { + return nil, fmt.Errorf("missing collector: %s", filter) + } + if !*enabled { + return nil, fmt.Errorf("disabled collector: %s", filter) + } + f[filter] = true + } + collectors := make(map[string]Collector) + initiatedCollectorsMtx.Lock() + defer initiatedCollectorsMtx.Unlock() + for key, enabled := range collectorState { + if !*enabled || (len(f) > 0 && !f[key]) { + continue + } + if collector, ok := initiatedCollectors[key]; ok { + collectors[key] = collector + } else { + collector, err := factories[key](e.esURL, e.httpClient) + if err != nil { + return nil, err + } + collectors[key] = collector + initiatedCollectors[key] = collector + } + } + + e.Collectors = collectors + + return e, nil +} + +func WithElasticsearchURL(esURL *url.URL) Option { + return func(e *ElasticsearchCollector) error { + e.esURL = esURL + return nil + } +} + +func WithHTTPClient(hc *http.Client) Option { + return func(e *ElasticsearchCollector) error { + e.httpClient = hc + return nil + } +} + +// Describe implements the prometheus.Collector interface. +func (e ElasticsearchCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- scrapeDurationDesc + ch <- scrapeSuccessDesc +} + +// Collect implements the prometheus.Collector interface. +func (e ElasticsearchCollector) Collect(ch chan<- prometheus.Metric) { + wg := sync.WaitGroup{} + ctx := context.TODO() + wg.Add(len(e.Collectors)) + for name, c := range e.Collectors { + go func(name string, c Collector) { + execute(ctx, name, c, ch) + wg.Done() + }(name, c) + } + wg.Wait() +} + +func execute(ctx context.Context, name string, c Collector, ch chan<- prometheus.Metric) { + begin := time.Now() + err := c.Update(ctx, ch) + duration := time.Since(begin) + var success float64 + + if err != nil { + if IsNoDataError(err) { + log.Printf("collector returned no data, name: %s, duration_seconds: %f, err: %s\n", name, duration.Seconds(), err) + } else { + log.Printf("collector failed, name: %s, duration_seconds: %f, err: %s\n", name, duration.Seconds(), err) + } + success = 0 + } else { + log.Printf("collector succeeded, name: %s, duration_seconds: %f, err: %s\n", name, duration.Seconds(), err) + success = 1 + } + ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, duration.Seconds(), name) + ch <- prometheus.MustNewConstMetric(scrapeSuccessDesc, prometheus.GaugeValue, success, name) +} + +// collectorFlagAction generates a new action function for the given collector +// to track whether it has been explicitly enabled or disabled from the command line. +// A new action function is needed for each collector flag because the ParseContext +// does not contain information about which flag called the action. +// See: https://github.com/alecthomas/kingpin/issues/294 +func collectorFlagAction(collector string) func(ctx *kingpin.ParseContext) error { + return func(ctx *kingpin.ParseContext) error { + forcedCollectors[collector] = true + return nil + } +} + +// ErrNoData indicates the collector found no data to collect, but had no other error. +var ErrNoData = errors.New("collector returned no data") + +func IsNoDataError(err error) bool { + return err == ErrNoData +} diff --git a/inputs/elasticsearch/collector/collector_test.go b/inputs/elasticsearch/collector/collector_test.go new file mode 100644 index 00000000..80c7fa5d --- /dev/null +++ b/inputs/elasticsearch/collector/collector_test.go @@ -0,0 +1,36 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" +) + +// wrapCollector is a util to let you test your Collector implementation. +// +// Use this with prometheus/client_golang/prometheus/testutil to test metric output, for example: +// +// testutil.CollectAndCompare(wrapCollector{c}, strings.NewReader(want)) +type wrapCollector struct { + c Collector +} + +func (w wrapCollector) Describe(_ chan<- *prometheus.Desc) { +} + +func (w wrapCollector) Collect(ch chan<- prometheus.Metric) { + w.c.Update(context.Background(), ch) +} diff --git a/inputs/elasticsearch/collector/data_stream.go b/inputs/elasticsearch/collector/data_stream.go new file mode 100644 index 00000000..15c6322b --- /dev/null +++ b/inputs/elasticsearch/collector/data_stream.go @@ -0,0 +1,146 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +type dataStreamMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(dataStreamStats DataStreamStatsDataStream) float64 + Labels func(dataStreamStats DataStreamStatsDataStream) []string +} + +var ( + defaultDataStreamLabels = []string{"data_stream"} + defaultDataStreamLabelValues = func(dataStreamStats DataStreamStatsDataStream) []string { + return []string{dataStreamStats.DataStream} + } +) + +// DataStream Information Struct +type DataStream struct { + client *http.Client + url *url.URL + + dataStreamMetrics []*dataStreamMetric +} + +// NewDataStream defines DataStream Prometheus metrics +func NewDataStream(client *http.Client, url *url.URL) *DataStream { + return &DataStream{ + client: client, + url: url, + + dataStreamMetrics: []*dataStreamMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "data_stream", "backing_indices_total"), + "Number of backing indices", + defaultDataStreamLabels, nil, + ), + Value: func(dataStreamStats DataStreamStatsDataStream) float64 { + return float64(dataStreamStats.BackingIndices) + }, + Labels: defaultDataStreamLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "data_stream", "store_size_bytes"), + "Store size of data stream", + defaultDataStreamLabels, nil, + ), + Value: func(dataStreamStats DataStreamStatsDataStream) float64 { + return float64(dataStreamStats.StoreSizeBytes) + }, + Labels: defaultDataStreamLabelValues, + }, + }, + } +} + +// Describe adds DataStream metrics descriptions +func (ds *DataStream) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range ds.dataStreamMetrics { + ch <- metric.Desc + } +} + +func (ds *DataStream) fetchAndDecodeDataStreamStats() (DataStreamStatsResponse, error) { + var dsr DataStreamStatsResponse + + u := *ds.url + u.Path = path.Join(u.Path, "/_data_stream/*/_stats") + res, err := ds.client.Get(u.String()) + if err != nil { + return dsr, fmt.Errorf("failed to get data stream stats health from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return dsr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return dsr, err + } + + if err := json.Unmarshal(bts, &dsr); err != nil { + return dsr, err + } + + return dsr, nil +} + +// Collect gets DataStream metric values +func (ds *DataStream) Collect(ch chan<- prometheus.Metric) { + + dataStreamStatsResp, err := ds.fetchAndDecodeDataStreamStats() + if err != nil { + log.Println("failed to fetch and decode data stream stats, err: ", err) + return + } + + for _, metric := range ds.dataStreamMetrics { + for _, dataStream := range dataStreamStatsResp.DataStreamStats { + fmt.Printf("Metric: %+v", dataStream) + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(dataStream), + metric.Labels(dataStream)..., + ) + } + } +} diff --git a/inputs/elasticsearch/collector/data_stream_response.go b/inputs/elasticsearch/collector/data_stream_response.go new file mode 100644 index 00000000..41365a09 --- /dev/null +++ b/inputs/elasticsearch/collector/data_stream_response.go @@ -0,0 +1,38 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +// DataStreamStatsResponse is a representation of the Data Stream stats +type DataStreamStatsResponse struct { + Shards DataStreamStatsShards `json:"_shards"` + DataStreamCount int64 `json:"data_stream_count"` + BackingIndices int64 `json:"backing_indices"` + TotalStoreSizeBytes int64 `json:"total_store_size_bytes"` + DataStreamStats []DataStreamStatsDataStream `json:"data_streams"` +} + +// DataStreamStatsShards defines data stream stats shards information structure +type DataStreamStatsShards struct { + Total int64 `json:"total"` + Successful int64 `json:"successful"` + Failed int64 `json:"failed"` +} + +// DataStreamStatsDataStream defines the structure of per data stream stats +type DataStreamStatsDataStream struct { + DataStream string `json:"data_stream"` + BackingIndices int64 `json:"backing_indices"` + StoreSizeBytes int64 `json:"store_size_bytes"` + MaximumTimestamp int64 `json:"maximum_timestamp"` +} diff --git a/inputs/elasticsearch/collector/data_stream_test.go b/inputs/elasticsearch/collector/data_stream_test.go new file mode 100644 index 00000000..6fdda5fe --- /dev/null +++ b/inputs/elasticsearch/collector/data_stream_test.go @@ -0,0 +1,77 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestDataStream(t *testing.T) { + + tests := []struct { + name string + file string + want string + }{ + { + name: "7.15.0", + file: "../fixtures/datastream/7.15.0.json", + want: `# HELP elasticsearch_data_stream_backing_indices_total Number of backing indices + # TYPE elasticsearch_data_stream_backing_indices_total counter + elasticsearch_data_stream_backing_indices_total{data_stream="bar"} 2 + elasticsearch_data_stream_backing_indices_total{data_stream="foo"} 5 + # HELP elasticsearch_data_stream_store_size_bytes Store size of data stream + # TYPE elasticsearch_data_stream_store_size_bytes counter + elasticsearch_data_stream_store_size_bytes{data_stream="bar"} 6.7382272e+08 + elasticsearch_data_stream_store_size_bytes{data_stream="foo"} 4.29205396e+08 + `, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + c := NewDataStream(http.DefaultClient, u) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/ilm_indices.go b/inputs/elasticsearch/collector/ilm_indices.go new file mode 100644 index 00000000..fcdc8d69 --- /dev/null +++ b/inputs/elasticsearch/collector/ilm_indices.go @@ -0,0 +1,145 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +type ilmMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(timeMillis float64) float64 + Labels []string +} + +// Index Lifecycle Management information object +type IlmIndiciesCollector struct { + client *http.Client + url *url.URL + + ilmMetric ilmMetric +} + +type IlmResponse struct { + Indices map[string]IlmIndexResponse `json:"indices"` +} + +type IlmIndexResponse struct { + Index string `json:"index"` + Managed bool `json:"managed"` + Phase string `json:"phase"` + Action string `json:"action"` + Step string `json:"step"` + StepTimeMillis float64 `json:"step_time_millis"` +} + +var ( + defaultIlmIndicesMappingsLabels = []string{"index", "phase", "action", "step"} +) + +// NewIlmIndicies defines Index Lifecycle Management Prometheus metrics +func NewIlmIndicies(client *http.Client, url *url.URL) *IlmIndiciesCollector { + subsystem := "ilm_index" + + return &IlmIndiciesCollector{ + client: client, + url: url, + + ilmMetric: ilmMetric{ + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "status"), + "Status of ILM policy for index", + defaultIlmIndicesMappingsLabels, nil), + Value: func(timeMillis float64) float64 { + return timeMillis + }, + }, + } +} + +// Describe adds metrics description +func (i *IlmIndiciesCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- i.ilmMetric.Desc +} + +func (i *IlmIndiciesCollector) fetchAndDecodeIlm() (IlmResponse, error) { + var ir IlmResponse + + u := *i.url + u.Path = path.Join(u.Path, "/_all/_ilm/explain") + + res, err := i.client.Get(u.String()) + if err != nil { + return ir, fmt.Errorf("failed to get index stats from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return ir, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return ir, err + } + + if err := json.Unmarshal(bts, &ir); err != nil { + return ir, err + } + + return ir, nil +} + +func bool2int(managed bool) float64 { + if managed { + return 1 + } + return 0 +} + +// Collect pulls metric values from Elasticsearch +func (i *IlmIndiciesCollector) Collect(ch chan<- prometheus.Metric) { + // indices + ilmResp, err := i.fetchAndDecodeIlm() + if err != nil { + log.Println("failed to fetch and decode ILM stats, err: ", err) + return + } + + for indexName, indexIlm := range ilmResp.Indices { + ch <- prometheus.MustNewConstMetric( + i.ilmMetric.Desc, + i.ilmMetric.Type, + i.ilmMetric.Value(bool2int(indexIlm.Managed)), + indexName, indexIlm.Phase, indexIlm.Action, indexIlm.Step, + ) + } +} diff --git a/inputs/elasticsearch/collector/ilm_indices_test.go b/inputs/elasticsearch/collector/ilm_indices_test.go new file mode 100644 index 00000000..2752562f --- /dev/null +++ b/inputs/elasticsearch/collector/ilm_indices_test.go @@ -0,0 +1,110 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestILMMetrics(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION + // curl -XPUT http://localhost:9200/twitter + // curl -X PUT "localhost:9200/_ilm/policy/my_policy?pretty" -H 'Content-Type: application/json' -d' + // { + // "policy": { + // "phases": { + // "warm": { + // "min_age": "10d", + // "actions": { + // "forcemerge": { + // "max_num_segments": 1 + // } + // } + // }, + // "delete": { + // "min_age": "30d", + // "actions": { + // "delete": {} + // } + // } + // } + // } + // } + // ' + // curl -X PUT "localhost:9200/facebook?pretty" -H 'Content-Type: application/json' -d' + // { + // "settings": { + // "index": { + // "lifecycle": { + // "name": "my_policy" + // } + // } + // } + // } + // ' + // curl http://localhost:9200/_all/_ilm/explain + tests := []struct { + name string + file string + want string + }{ + { + name: "6.6.0", + file: "../fixtures/ilm_indices/6.6.0.json", + want: ` +# HELP elasticsearch_ilm_index_status Status of ILM policy for index +# TYPE elasticsearch_ilm_index_status gauge +elasticsearch_ilm_index_status{action="",index="twitter",phase="",step=""} 0 +elasticsearch_ilm_index_status{action="complete",index="facebook",phase="new",step="complete"} 1 + `, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + c := NewIlmIndicies(http.DefaultClient, u) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/ilm_status.go b/inputs/elasticsearch/collector/ilm_status.go new file mode 100644 index 00000000..f8d2e8e8 --- /dev/null +++ b/inputs/elasticsearch/collector/ilm_status.go @@ -0,0 +1,132 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + ilmStatuses = []string{"STOPPED", "RUNNING", "STOPPING"} +) + +type ilmStatusMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(ilm *IlmStatusResponse, status string) float64 + Labels func(status string) []string +} + +// IlmStatusCollector information struct +type IlmStatusCollector struct { + client *http.Client + url *url.URL + + metric ilmStatusMetric +} + +type IlmStatusResponse struct { + OperationMode string `json:"operation_mode"` +} + +// NewIlmStatus defines Indices IndexIlms Prometheus metrics +func NewIlmStatus(client *http.Client, url *url.URL) *IlmStatusCollector { + subsystem := "ilm" + + return &IlmStatusCollector{ + client: client, + url: url, + + metric: ilmStatusMetric{ + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "status"), + "Current status of ilm. Status can be STOPPED, RUNNING, STOPPING.", + []string{"operation_mode"}, nil, + ), + Value: func(ilm *IlmStatusResponse, status string) float64 { + if ilm.OperationMode == status { + return 1 + } + return 0 + }, + }, + } +} + +// Describe add Snapshots metrics descriptions +func (im *IlmStatusCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- im.metric.Desc +} + +func (im *IlmStatusCollector) fetchAndDecodeIlm() (*IlmStatusResponse, error) { + u := *im.url + u.Path = path.Join(im.url.Path, "/_ilm/status") + + res, err := im.client.Get(u.String()) + if err != nil { + return nil, fmt.Errorf("failed to get from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + body, err := io.ReadAll(res.Body) + if err != nil { + log.Println("failed to read response body, err: ", err) + return nil, err + } + + err = res.Body.Close() + if err != nil { + log.Println("failed to close response body, err: ", err) + return nil, err + } + + var imr IlmStatusResponse + if err := json.Unmarshal(body, &imr); err != nil { + return nil, err + } + + return &imr, nil +} + +// Collect gets all indices Ilms metric values +func (im *IlmStatusCollector) Collect(ch chan<- prometheus.Metric) { + indicesIlmsResponse, err := im.fetchAndDecodeIlm() + if err != nil { + log.Println("failed to fetch and decode cluster ilm status, err: ", err) + return + } + + for _, status := range ilmStatuses { + ch <- prometheus.MustNewConstMetric( + im.metric.Desc, + im.metric.Type, + im.metric.Value(indicesIlmsResponse, status), + status, + ) + } + +} diff --git a/inputs/elasticsearch/collector/ilm_status_test.go b/inputs/elasticsearch/collector/ilm_status_test.go new file mode 100644 index 00000000..cb080911 --- /dev/null +++ b/inputs/elasticsearch/collector/ilm_status_test.go @@ -0,0 +1,78 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestILMStatus(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION + // curl http://localhost:9200/_ilm/status + tests := []struct { + name string + file string + want string + }{ + { + name: "6.6.0", + file: "../fixtures/ilm_status/6.6.0.json", + want: ` +# HELP elasticsearch_ilm_status Current status of ilm. Status can be STOPPED, RUNNING, STOPPING. +# TYPE elasticsearch_ilm_status gauge +elasticsearch_ilm_status{operation_mode="RUNNING"} 1 +elasticsearch_ilm_status{operation_mode="STOPPED"} 0 +elasticsearch_ilm_status{operation_mode="STOPPING"} 0 + `, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + c := NewIlmStatus(http.DefaultClient, u) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/indices.go b/inputs/elasticsearch/collector/indices.go new file mode 100644 index 00000000..8fb55058 --- /dev/null +++ b/inputs/elasticsearch/collector/indices.go @@ -0,0 +1,2554 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "sort" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" + + "flashcat.cloud/categraf/inputs/elasticsearch/pkg/clusterinfo" +) + +type labels struct { + keys func(...string) []string + values func(*clusterinfo.Response, ...string) []string +} + +type indexMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(indexStats IndexStatsIndexResponse) float64 + Labels labels +} + +type shardMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(data IndexStatsIndexShardsDetailResponse) float64 + Labels labels +} + +type aliasMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func() float64 + Labels labels +} + +// Indices information struct +type Indices struct { + client *http.Client + url *url.URL + shards bool + aliases bool + indicesIncluded []string + clusterInfoCh chan *clusterinfo.Response + lastClusterInfo *clusterinfo.Response + + up prometheus.Gauge + totalScrapes prometheus.Counter + jsonParseFailures prometheus.Counter + + indexMetrics []*indexMetric + shardMetrics []*shardMetric + aliasMetrics []*aliasMetric +} + +// NewIndices defines Indices Prometheus metrics +func NewIndices(client *http.Client, url *url.URL, shards bool, includeAliases bool, indicesIncluded []string) *Indices { + + indexLabels := labels{ + keys: func(...string) []string { + return []string{"index", "cluster"} + }, + values: func(lastClusterinfo *clusterinfo.Response, s ...string) []string { + if lastClusterinfo != nil { + return append(s, lastClusterinfo.ClusterName) + } + // this shouldn't happen, as the clusterinfo Retriever has a blocking + // Run method. It blocks until the first clusterinfo call has succeeded + return append(s, "unknown_cluster") + }, + } + + shardLabels := labels{ + keys: func(...string) []string { + return []string{"index", "shard", "node", "primary", "cluster"} + }, + values: func(lastClusterinfo *clusterinfo.Response, s ...string) []string { + if lastClusterinfo != nil { + return append(s, lastClusterinfo.ClusterName) + } + // this shouldn't happen, as the clusterinfo Retriever has a blocking + // Run method. It blocks until the first clusterinfo call has succeeded + return append(s, "unknown_cluster") + }, + } + + aliasLabels := labels{ + keys: func(...string) []string { + return []string{"index", "alias", "cluster"} + }, + values: func(lastClusterinfo *clusterinfo.Response, s ...string) []string { + if lastClusterinfo != nil { + return append(s, lastClusterinfo.ClusterName) + } + // this shouldn't happen, as the clusterinfo Retriever has a blocking + // Run method. It blocks until the first clusterinfo call has succeeded + return append(s, "unknown_cluster") + }, + } + + indices := &Indices{ + client: client, + url: url, + shards: shards, + aliases: includeAliases, + indicesIncluded: indicesIncluded, + clusterInfoCh: make(chan *clusterinfo.Response), + lastClusterInfo: &clusterinfo.Response{ + ClusterName: "unknown_cluster", + }, + + up: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "index_stats", "up"), + Help: "Was the last scrape of the Elasticsearch index endpoint successful.", + }), + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "index_stats", "total_scrapes"), + Help: "Current total Elasticsearch index scrapes.", + }), + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "index_stats", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + + indexMetrics: []*indexMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "docs_count"), + "Total count of documents", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Docs.Count) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "docs_deleted"), + "Total count of deleted documents", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Docs.Deleted) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "store_size_in_bytes"), + "Current total size of stored index data in bytes with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Store.SizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "throttle_time_seconds"), + "Total time the index has been throttled in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Store.ThrottleTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_count"), + "Current number of segments with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.Count) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_memory_in_bytes"), + "Current size of segments with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.MemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_terms_memory_in_bytes"), + "Current number of terms with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.TermsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_stored_fields_memory_in_bytes"), + "Current size of fields with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.StoredFieldsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_term_vectors_memory_in_bytes"), + "Current size of term vectors with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.TermVectorsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_norms_memory_in_bytes"), + "Current size of norms with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.NormsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_points_memory_in_bytes"), + "Current size of points with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.PointsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_doc_values_memory_in_bytes"), + "Current size of doc values with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.DocValuesMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_index_writer_memory_in_bytes"), + "Current size of index writer with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.IndexWriterMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_version_map_memory_in_bytes"), + "Current size of version map with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.VersionMapMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_fixed_bit_set_memory_in_bytes"), + "Current size of fixed bit with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.FixedBitSetMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "segments_max_unsafe_auto_id_timestamp"), + "Current max unsafe auto id timestamp with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Segments.MaxUnsafeAutoIDTimestamp) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "translog_earliest_last_modified_age"), + "Current earliest last modified age with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Translog.EarliestLastModifiedAge) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "translog_operations"), + "Current number of operations in the transaction log with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Translog.Operations) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "translog_size_in_bytes"), + "Current size of transaction log with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Translog.SizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "translog_uncommitted_operations"), + "Current number of uncommitted operations in the transaction log with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Translog.UncommittedOperations) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "translog_uncommitted_size_in_bytes"), + "Current size of uncommitted transaction log with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Translog.UncommittedSizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "completion_size_in_bytes"), + "Current size of completion with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Completion.SizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_query_time_seconds"), + "Total search query time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.QueryTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_query_current"), + "The number of currently active queries", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.QueryCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_open_contexts"), + "Total number of open search contexts", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.OpenContexts) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_query_total"), + "Total number of queries", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.QueryTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_fetch_time_seconds"), + "Total search fetch time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.FetchTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_fetch"), + "Total search fetch count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.FetchTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_fetch_current"), + "Current search fetch count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.FetchCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_scroll_time_seconds"), + "Total search scroll time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.ScrollTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_scroll_current"), + "Current search scroll count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.ScrollCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_scroll"), + "Total search scroll count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.ScrollTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_suggest_time_seconds"), + "Total search suggest time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.SuggestTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_suggest_total"), + "Total search suggest count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.SuggestTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "search_suggest_current"), + "Current search suggest count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Search.SuggestCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "indexing_index_time_seconds"), + "Total indexing index time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.IndexTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "index_current"), + "The number of documents currently being indexed to an index", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.IndexCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "index_failed"), + "Total indexing index failed count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.IndexFailed) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "delete_current"), + "The number of delete operations currently being processed", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.DeleteCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "indexing_index"), + "Total indexing index count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.IndexTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "indexing_delete_time_seconds"), + "Total indexing delete time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.DeleteTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "indexing_delete"), + "Total indexing delete count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.DeleteTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "indexing_noop_update"), + "Total indexing no-op update count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.NoopUpdateTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "indexing_throttle_time_seconds"), + "Total indexing throttle time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Indexing.ThrottleTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "get_time_seconds"), + "Total get time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Get.TimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "get_exists_total"), + "Total exists count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Get.ExistsTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "get_exists_time_seconds"), + "Total exists time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Get.ExistsTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "get_total"), + "Total get count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Get.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "get_missing_total"), + "Total missing count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Get.MissingTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "get_missing_time_seconds"), + "Total missing time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Get.MissingTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "get_current"), + "Current get count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Get.Current) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_time_seconds"), + "Total merge time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_total"), + "Total merge count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_total_docs"), + "Total merge docs count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.TotalDocs) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_total_size_in_bytes"), + "Total merge size in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.TotalSizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_current"), + "Current merge count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.Current) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_current_docs"), + "Current merge docs count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.CurrentDocs) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_current_size_in_bytes"), + "Current merge size in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.CurrentSizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_total_throttle_time_seconds"), + "Total merge I/O throttle time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.TotalThrottledTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_total_stopped_time_seconds"), + "Total large merge stopped time in seconds, allowing smaller merges to complete", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.TotalStoppedTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "merges_total_auto_throttle_bytes"), + "Total bytes that were auto-throttled during merging", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Merges.TotalAutoThrottleInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "refresh_external_total_time_seconds"), + "Total external refresh time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Refresh.ExternalTotalTime) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "refresh_external_total"), + "Total external refresh count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Refresh.ExternalTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "refresh_total_time_seconds"), + "Total refresh time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Refresh.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "refresh_total"), + "Total refresh count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Refresh.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "refresh_listeners"), + "Total number of refresh listeners", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Refresh.Listeners) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "recovery_current_as_source"), + "Current number of recovery as source", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Recovery.CurrentAsSource) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "recovery_current_as_target"), + "Current number of recovery as target", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Recovery.CurrentAsTarget) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "recovery_throttle_time_seconds"), + "Total recovery throttle time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Recovery.ThrottleTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "flush_time_seconds_total"), + "Total flush time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Flush.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "flush_total"), + "Total flush count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Flush.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "flush_periodic"), + "Total periodic flush count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Flush.Periodic) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "warmer_time_seconds_total"), + "Total warmer time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Warmer.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "warmer_total"), + "Total warmer count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Warmer.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "query_cache_memory_in_bytes"), + "Total query cache memory bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.QueryCache.MemorySizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "query_cache_size"), + "Total query cache size", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.QueryCache.CacheSize) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "query_cache_total_count"), + "Total query cache count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.QueryCache.TotalCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "query_cache_hit_count"), + "Total query cache hits count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.QueryCache.HitCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "query_cache_miss_count"), + "Total query cache misses count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.QueryCache.MissCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "query_cache_cache_count"), + "Total query cache caches count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.QueryCache.CacheCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "query_cache_evictions"), + "Total query cache evictions count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.QueryCache.Evictions) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "request_cache_memory_in_bytes"), + "Total request cache memory bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.RequestCache.MemorySizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "request_cache_hit_count"), + "Total request cache hits count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.RequestCache.HitCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "request_cache_miss_count"), + "Total request cache misses count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.RequestCache.MissCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "request_cache_evictions"), + "Total request cache evictions count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.RequestCache.Evictions) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "fielddata_memory_in_bytes"), + "Total fielddata memory bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Fielddata.MemorySizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "fielddata_evictions"), + "Total fielddata evictions count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Fielddata.Evictions) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "seq_no_global_checkpoint"), + "Global checkpoint", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Seq.GlobalCheckpoint) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "seq_no_local_checkpoint"), + "Local checkpoint", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Seq.LocalCheckpoint) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_total", "seq_no_max_seq_no"), + "Max sequence number", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Seq.MaxSeqNo) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "docs_count"), + "Total count of documents", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Docs.Count) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "docs_deleted"), + "Total count of deleted documents", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Docs.Deleted) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "store_size_in_bytes"), + "Current total size of stored index data in bytes with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Store.SizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "throttle_time_seconds"), + "Total time the index has been throttled in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Store.ThrottleTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_count"), + "Current number of segments with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.Count) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_memory_in_bytes"), + "Current size of segments with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.MemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_terms_memory_in_bytes"), + "Current number of terms with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.TermsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_stored_fields_memory_in_bytes"), + "Current size of fields with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.StoredFieldsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_term_vectors_memory_in_bytes"), + "Current size of term vectors with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.TermVectorsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_norms_memory_in_bytes"), + "Current size of norms with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.NormsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_points_memory_in_bytes"), + "Current size of points with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.PointsMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_doc_values_memory_in_bytes"), + "Current size of doc values with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.DocValuesMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_index_writer_memory_in_bytes"), + "Current size of index writer with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.IndexWriterMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_version_map_memory_in_bytes"), + "Current size of version map with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.VersionMapMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_fixed_bit_set_memory_in_bytes"), + "Current size of fixed bit with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.FixedBitSetMemoryInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "segments_max_unsafe_auto_id_timestamp"), + "Current max unsafe auto id timestamp with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Segments.MaxUnsafeAutoIDTimestamp) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "translog_earliest_last_modified_age"), + "Current earliest last modified age with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Translog.EarliestLastModifiedAge) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "translog_operations"), + "Current number of operations in the transaction log with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Translog.Operations) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "translog_size_in_bytes"), + "Current size of transaction log with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Translog.SizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "translog_uncommitted_operations"), + "Current number of uncommitted operations in the transaction log with all shards on all nodes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Translog.UncommittedOperations) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "translog_uncommitted_size_in_bytes"), + "Current size of uncommitted transaction log with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Translog.UncommittedSizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "completion_size_in_bytes"), + "Current size of completion with all shards on all nodes in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Completion.SizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_query_time_seconds"), + "Total search query time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.QueryTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_query_current"), + "The number of currently active queries", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.QueryCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_open_contexts"), + "Total number of open search contexts", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.OpenContexts) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_query_total"), + "Total number of queries", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.QueryTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_fetch_time_seconds"), + "Total search fetch time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.FetchTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_fetch"), + "Total search fetch count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.FetchTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_fetch_current"), + "Current search fetch count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.FetchCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_scroll_time_seconds"), + "Total search scroll time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.ScrollTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_scroll_current"), + "Current search scroll count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.ScrollCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_scroll"), + "Total search scroll count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.ScrollTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_suggest_time_seconds"), + "Total search suggest time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.SuggestTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_suggest_total"), + "Total search suggest count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.SuggestTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "search_suggest_current"), + "Current search suggest count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Search.SuggestCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "indexing_index_time_seconds"), + "Total indexing index time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.IndexTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "index_current"), + "The number of documents currently being indexed to an index", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.IndexCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "index_failed"), + "Total indexing index failed count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.IndexFailed) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "delete_current"), + "The number of delete operations currently being processed", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.DeleteCurrent) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "indexing_index"), + "Total indexing index count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.IndexTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "indexing_delete_time_seconds"), + "Total indexing delete time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.DeleteTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "indexing_delete"), + "Total indexing delete count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.DeleteTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "indexing_noop_update"), + "Total indexing no-op update count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.NoopUpdateTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "indexing_throttle_time_seconds"), + "Total indexing throttle time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Indexing.ThrottleTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "get_time_seconds"), + "Total get time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Get.TimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "get_exists_total"), + "Total exists count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Get.ExistsTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "get_exists_time_seconds"), + "Total exists time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Get.ExistsTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "get_total"), + "Total get count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Get.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "get_missing_total"), + "Total missing count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Get.MissingTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "get_missing_time_seconds"), + "Total missing time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Get.MissingTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "get_current"), + "Current get count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Get.Current) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_time_seconds"), + "Total merge time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_total"), + "Total merge count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_total_docs"), + "Total merge docs count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.TotalDocs) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_total_size_in_bytes"), + "Total merge size in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.TotalSizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_current"), + "Current merge count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.Current) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_current_docs"), + "Current merge docs count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.CurrentDocs) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_current_size_in_bytes"), + "Current merge size in bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.CurrentSizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_total_throttle_time_seconds"), + "Total merge I/O throttle time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.TotalThrottledTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_total_stopped_time_seconds"), + "Total large merge stopped time in seconds, allowing smaller merges to complete", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.TotalStoppedTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "merges_total_auto_throttle_bytes"), + "Total bytes that were auto-throttled during merging", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Merges.TotalAutoThrottleInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "refresh_external_total_time_seconds"), + "Total external refresh time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Refresh.ExternalTotalTime) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "refresh_external_total"), + "Total external refresh count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Refresh.ExternalTotal) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "refresh_total_time_seconds"), + "Total refresh time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Refresh.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "refresh_total"), + "Total refresh count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Refresh.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "refresh_listeners"), + "Total number of refresh listeners", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Refresh.Listeners) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "recovery_current_as_source"), + "Current number of recovery as source", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Recovery.CurrentAsSource) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "recovery_current_as_target"), + "Current number of recovery as target", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Recovery.CurrentAsTarget) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "recovery_throttle_time_seconds"), + "Total recovery throttle time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Recovery.ThrottleTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "flush_time_seconds_total"), + "Total flush time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Flush.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "flush_total"), + "Total flush count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Flush.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "flush_periodic"), + "Total periodic flush count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Flush.Periodic) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "warmer_time_seconds_total"), + "Total warmer time in seconds", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Warmer.TotalTimeInMillis) / 1000 + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "warmer_total"), + "Total warmer count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Warmer.Total) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "query_cache_memory_in_bytes"), + "Total query cache memory bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.QueryCache.MemorySizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "query_cache_size"), + "Total query cache size", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.QueryCache.CacheSize) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "query_cache_total_count"), + "Total query cache count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.QueryCache.TotalCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "query_cache_hit_count"), + "Total query cache hits count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.QueryCache.HitCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "query_cache_miss_count"), + "Total query cache misses count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.QueryCache.MissCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "query_cache_cache_count"), + "Total query cache caches count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.QueryCache.CacheCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "query_cache_evictions"), + "Total query cache evictions count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.QueryCache.Evictions) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "request_cache_memory_in_bytes"), + "Total request cache memory bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.RequestCache.MemorySizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "request_cache_hit_count"), + "Total request cache hits count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.RequestCache.HitCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "request_cache_miss_count"), + "Total request cache misses count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.RequestCache.MissCount) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "request_cache_evictions"), + "Total request cache evictions count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.RequestCache.Evictions) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "fielddata_memory_in_bytes"), + "Total fielddata memory bytes", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Fielddata.MemorySizeInBytes) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "fielddata_evictions"), + "Total fielddata evictions count", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Fielddata.Evictions) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "seq_no_global_checkpoint"), + "Global checkpoint", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Seq.GlobalCheckpoint) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "seq_no_local_checkpoint"), + "Local checkpoint", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Seq.LocalCheckpoint) + }, + Labels: indexLabels, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats_primaries", "seq_no_max_seq_no"), + "Max sequence number", + indexLabels.keys(), nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Seq.MaxSeqNo) + }, + Labels: indexLabels, + }, + }, + shardMetrics: []*shardMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats", "shards_docs"), + "Count of documents on this shard", + shardLabels.keys(), nil, + ), + Value: func(data IndexStatsIndexShardsDetailResponse) float64 { + return float64(data.Docs.Count) + }, + Labels: shardLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats", "shards_docs_deleted"), + "Count of deleted documents on this shard", + shardLabels.keys(), nil, + ), + Value: func(data IndexStatsIndexShardsDetailResponse) float64 { + return float64(data.Docs.Deleted) + }, + Labels: shardLabels, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats", "shards_store_size_in_bytes"), + "Store size of this shard", + shardLabels.keys(), nil, + ), + Value: func(data IndexStatsIndexShardsDetailResponse) float64 { + return float64(data.Store.SizeInBytes) + }, + Labels: shardLabels, + }, + }, + + aliasMetrics: []*aliasMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_stats", "aliases"), + "Record aliases associated with an index", + aliasLabels.keys(), nil, + ), + Value: func() float64 { + return float64(1) + }, + Labels: aliasLabels, + }, + }, + } + + // start go routine to fetch clusterinfo updates and save them to lastClusterinfo + go func() { + for ci := range indices.clusterInfoCh { + if ci != nil { + log.Println("received cluster info update, cluster: ", ci.ClusterName) + indices.lastClusterInfo = ci + } + } + }() + return indices +} + +// ClusterLabelUpdates returns a pointer to a channel to receive cluster info updates. It implements the +// (not exported) clusterinfo.consumer interface +func (i *Indices) ClusterLabelUpdates() *chan *clusterinfo.Response { + return &i.clusterInfoCh +} + +// String implements the stringer interface. It is part of the clusterinfo.consumer interface +func (i *Indices) String() string { + return namespace + "indices" +} + +// Describe add Indices metrics descriptions +func (i *Indices) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range i.indexMetrics { + ch <- metric.Desc + } + ch <- i.up.Desc() + ch <- i.totalScrapes.Desc() + ch <- i.jsonParseFailures.Desc() +} + +func (i *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { + var isr indexStatsResponse + + u := *i.url + if len(i.indicesIncluded) == 0 { + u.Path = path.Join(u.Path, "/_all/_stats") + } else { + u.Path = path.Join(u.Path, "/"+strings.Join(i.indicesIncluded, ",")+"/_stats") + } + if i.shards { + u.RawQuery = "ignore_unavailable=true&level=shards" + } else { + u.RawQuery = "ignore_unavailable=true" + } + + bts, err := i.queryURL(&u) + if err != nil { + return isr, err + } + + if err := json.Unmarshal(bts, &isr); err != nil { + i.jsonParseFailures.Inc() + return isr, err + } + + if i.aliases { + isr.Aliases = map[string][]string{} + asr, err := i.fetchAndDecodeAliases() + if err != nil { + log.Println("err: ", err.Error()) + return isr, err + } + + for indexName, aliases := range asr { + var aliasList []string + for aliasName := range aliases.Aliases { + aliasList = append(aliasList, aliasName) + } + + if len(aliasList) > 0 { + sort.Strings(aliasList) + isr.Aliases[indexName] = aliasList + } + } + } + + return isr, nil +} + +func (i *Indices) fetchAndDecodeAliases() (aliasesResponse, error) { + var asr aliasesResponse + + u := *i.url + u.Path = path.Join(u.Path, "/_alias") + + bts, err := i.queryURL(&u) + if err != nil { + return asr, err + } + + if err := json.Unmarshal(bts, &asr); err != nil { + i.jsonParseFailures.Inc() + return asr, err + } + + return asr, nil +} + +func (i *Indices) queryURL(u *url.URL) ([]byte, error) { + res, err := i.client.Get(u.String()) + if err != nil { + return []byte{}, fmt.Errorf("failed to get resource from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return []byte{}, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return []byte{}, err + } + + return bts, nil +} + +// Collect gets Indices metric values +func (i *Indices) Collect(ch chan<- prometheus.Metric) { + i.totalScrapes.Inc() + defer func() { + ch <- i.up + ch <- i.totalScrapes + ch <- i.jsonParseFailures + }() + + // indices + indexStatsResp, err := i.fetchAndDecodeIndexStats() + if err != nil { + i.up.Set(0) + log.Println("failed to fetch and decode index stats, err", err) + return + } + i.up.Set(1) + + // Alias stats + if i.aliases { + for _, metric := range i.aliasMetrics { + for indexName, aliases := range indexStatsResp.Aliases { + for _, alias := range aliases { + labelValues := metric.Labels.values(i.lastClusterInfo, indexName, alias) + + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(), + labelValues..., + ) + } + } + } + } + + // Index stats + for indexName, indexStats := range indexStatsResp.Indices { + for _, metric := range i.indexMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(indexStats), + metric.Labels.values(i.lastClusterInfo, indexName)..., + ) + + } + if i.shards { + for _, metric := range i.shardMetrics { + // gaugeVec := prometheus.NewGaugeVec(metric.Opts, metric.Labels) + for shardNumber, shards := range indexStats.Shards { + for _, shard := range shards { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(shard), + metric.Labels.values(i.lastClusterInfo, indexName, shardNumber, shard.Routing.Node, strconv.FormatBool(shard.Routing.Primary))..., + ) + } + } + } + } + } +} diff --git a/inputs/elasticsearch/collector/indices_mappings.go b/inputs/elasticsearch/collector/indices_mappings.go new file mode 100644 index 00000000..3f844d61 --- /dev/null +++ b/inputs/elasticsearch/collector/indices_mappings.go @@ -0,0 +1,159 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + defaultIndicesMappingsLabels = []string{"index"} +) + +type indicesMappingsMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(indexMapping IndexMapping) float64 +} + +// IndicesMappings information struct +type IndicesMappings struct { + client *http.Client + url *url.URL + + metrics []*indicesMappingsMetric +} + +// NewIndicesMappings defines Indices IndexMappings Prometheus metrics +func NewIndicesMappings(client *http.Client, url *url.URL) *IndicesMappings { + subsystem := "indices_mappings_stats" + + return &IndicesMappings{ + client: client, + url: url, + + metrics: []*indicesMappingsMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "fields"), + "Current number fields within cluster.", + defaultIndicesMappingsLabels, nil, + ), + Value: func(indexMapping IndexMapping) float64 { + return countFieldsRecursive(indexMapping.Mappings.Properties, 0) + }, + }, + }, + } +} + +func countFieldsRecursive(properties IndexMappingProperties, fieldCounter float64) float64 { + // iterate over all properties + for _, property := range properties { + + if property.Type != nil && *property.Type != "object" { + // property has a type set - counts as a field unless the value is object + // as the recursion below will handle counting that + fieldCounter++ + + // iterate over all fields of that property + for _, field := range property.Fields { + // field has a type set - counts as a field + if field.Type != nil { + fieldCounter++ + } + } + } + + // count recursively in case the property has more properties + if property.Properties != nil { + fieldCounter = 1 + countFieldsRecursive(property.Properties, fieldCounter) + } + } + + return fieldCounter +} + +// Describe add Snapshots metrics descriptions +func (im *IndicesMappings) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range im.metrics { + ch <- metric.Desc + } +} + +func (im *IndicesMappings) getAndParseURL(u *url.URL) (*IndicesMappingsResponse, error) { + res, err := im.client.Get(u.String()) + if err != nil { + return nil, fmt.Errorf("failed to get from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + body, err := io.ReadAll(res.Body) + if err != nil { + log.Println("failed to read response body, err: ", err) + return nil, err + } + + err = res.Body.Close() + if err != nil { + log.Println("failed to close response body, err: ", err) + return nil, err + } + + var imr IndicesMappingsResponse + if err := json.Unmarshal(body, &imr); err != nil { + return nil, err + } + + return &imr, nil +} + +func (im *IndicesMappings) fetchAndDecodeIndicesMappings() (*IndicesMappingsResponse, error) { + u := *im.url + u.Path = path.Join(u.Path, "/_all/_mappings") + return im.getAndParseURL(&u) +} + +// Collect gets all indices mappings metric values +func (im *IndicesMappings) Collect(ch chan<- prometheus.Metric) { + indicesMappingsResponse, err := im.fetchAndDecodeIndicesMappings() + if err != nil { + log.Println("failed to fetch and decode cluster mappings stats, err: ", err) + return + } + + for _, metric := range im.metrics { + for indexName, mappings := range *indicesMappingsResponse { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(mappings), + indexName, + ) + } + } +} diff --git a/inputs/elasticsearch/collector/indices_mappings_response.go b/inputs/elasticsearch/collector/indices_mappings_response.go new file mode 100644 index 00000000..3c54b3f7 --- /dev/null +++ b/inputs/elasticsearch/collector/indices_mappings_response.go @@ -0,0 +1,47 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +// IndicesMappingsResponse is a representation of elasticsearch mappings for each index +type IndicesMappingsResponse map[string]IndexMapping + +// IndexMapping defines the struct of the tree for the mappings of each index +type IndexMapping struct { + Mappings IndexMappings `json:"mappings"` +} + +// IndexMappings defines all index mappings +type IndexMappings struct { + Properties IndexMappingProperties `json:"properties"` +} + +// IndexMappingProperties defines all the properties of the current mapping +type IndexMappingProperties map[string]*IndexMappingProperty + +// IndexMappingFields defines all the fields of the current mapping +type IndexMappingFields map[string]*IndexMappingField + +// IndexMappingProperty defines a single property of the current index properties +type IndexMappingProperty struct { + Type *string `json:"type"` + Properties IndexMappingProperties `json:"properties"` + Fields IndexMappingFields `json:"fields"` +} + +// IndexMappingField defines a single property of the current index field +type IndexMappingField struct { + Type *string `json:"type"` + Properties IndexMappingProperties `json:"properties"` + Fields IndexMappingFields `json:"fields"` +} diff --git a/inputs/elasticsearch/collector/indices_mappings_test.go b/inputs/elasticsearch/collector/indices_mappings_test.go new file mode 100644 index 00000000..cc94c985 --- /dev/null +++ b/inputs/elasticsearch/collector/indices_mappings_test.go @@ -0,0 +1,125 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestMapping(t *testing.T) { + // Testcases created using: + // docker run -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.8.0 + // curl -XPUT http://localhost:9200/twitter + // curl -XPUT http://localhost:9200/facebook + /* curl -XPUT http://localhost:9200/twitter/_mapping -H 'Content-Type: application/json' -d'{ + "properties": { + "email": { + "type": "keyword" + }, + "phone": { + "type": "keyword" + } + } + }'*/ + /* curl -XPUT http://localhost:9200/facebook/_mapping -H 'Content-Type: application/json' -d'{ + "properties": { + "name": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "contact": { + "properties": { + "email": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "phone": { + "type": "text" + } + } + } + } + }'*/ + // curl http://localhost:9200/_all/_mapping + tests := []struct { + name string + file string + want string + }{ + { + name: "7.8.0", + file: "../fixtures/indices_mappings/7.8.0.json", + want: ` +# HELP elasticsearch_indices_mappings_stats_fields Current number fields within cluster. +# TYPE elasticsearch_indices_mappings_stats_fields gauge +elasticsearch_indices_mappings_stats_fields{index="facebook"} 6 +elasticsearch_indices_mappings_stats_fields{index="twitter"} 2 + `, + }, + { + name: "counts", + file: "../fixtures/indices_mappings/counts.json", + want: ` +# HELP elasticsearch_indices_mappings_stats_fields Current number fields within cluster. +# TYPE elasticsearch_indices_mappings_stats_fields gauge +elasticsearch_indices_mappings_stats_fields{index="test-data-2023.01.20"} 40 + `, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + c := NewIndicesMappings(http.DefaultClient, u) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/indices_response.go b/inputs/elasticsearch/collector/indices_response.go new file mode 100644 index 00000000..232d2402 --- /dev/null +++ b/inputs/elasticsearch/collector/indices_response.go @@ -0,0 +1,237 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +// indexStatsResponse is a representation of a Elasticsearch Index Stats +type indexStatsResponse struct { + Shards IndexStatsShardsResponse `json:"_shards"` + All IndexStatsIndexResponse `json:"_all"` + Indices map[string]IndexStatsIndexResponse `json:"indices"` + Aliases map[string][]string +} + +// aliasesResponse is a representation of a Elasticsearch Alias Query +type aliasesResponse map[string]aliasMapping + +// aliasMapping is a mapping of index names to a map of associated alias names where the alias names are keys +type aliasMapping struct { + Aliases map[string]map[string]interface{} `json:"aliases"` +} + +// IndexStatsShardsResponse defines index stats shards information structure +type IndexStatsShardsResponse struct { + Total int64 `json:"total"` + Successful int64 `json:"successful"` + Failed int64 `json:"failed"` +} + +// IndexStatsIndexResponse defines index stats index information structure +type IndexStatsIndexResponse struct { + Primaries IndexStatsIndexDetailResponse `json:"primaries"` + Total IndexStatsIndexDetailResponse `json:"total"` + Shards map[string][]IndexStatsIndexShardsDetailResponse `json:"shards"` +} + +// IndexStatsIndexDetailResponse defines index stats index details information structure +type IndexStatsIndexDetailResponse struct { + Docs IndexStatsIndexDocsResponse `json:"docs"` + Store IndexStatsIndexStoreResponse `json:"store"` + Seq IndexStataIndexSeqResponse `json:"seq_no"` + Indexing IndexStatsIndexIndexingResponse `json:"indexing"` + Get IndexStatsIndexGetResponse `json:"get"` + Search IndexStatsIndexSearchResponse `json:"search"` + Merges IndexStatsIndexMergesResponse `json:"merges"` + Refresh IndexStatsIndexRefreshResponse `json:"refresh"` + Flush IndexStatsIndexFlushResponse `json:"flush"` + Warmer IndexStatsIndexWarmerResponse `json:"warmer"` + QueryCache IndexStatsIndexQueryCacheResponse `json:"query_cache"` + Fielddata IndexStatsIndexFielddataResponse `json:"fielddata"` + Completion IndexStatsIndexCompletionResponse `json:"completion"` + Segments IndexStatsIndexSegmentsResponse `json:"segments"` + Translog IndexStatsIndexTranslogResponse `json:"translog"` + RequestCache IndexStatsIndexRequestCacheResponse `json:"request_cache"` + Recovery IndexStatsIndexRecoveryResponse `json:"recovery"` +} + +// IndexStatsIndexShardsDetailResponse defines index stats index shard details information structure +type IndexStatsIndexShardsDetailResponse struct { + *IndexStatsIndexDetailResponse + Routing IndexStatsIndexRoutingResponse `json:"routing"` +} + +// IndexStataIndexSeqResponse defines index stats index seq_no information structure +type IndexStataIndexSeqResponse struct { + GlobalCheckpoint int64 `json:"global_checkpoint"` + LocalCheckpoint int64 `json:"local_checkpoint"` + MaxSeqNo int64 `json:"max_seq_no"` +} + +// IndexStatsIndexRoutingResponse defines index stats index routing information structure +type IndexStatsIndexRoutingResponse struct { + Node string `json:"node"` + Primary bool `json:"primary"` +} + +// IndexStatsIndexDocsResponse defines index stats index documents information structure +type IndexStatsIndexDocsResponse struct { + Count int64 `json:"count"` + Deleted int64 `json:"deleted"` +} + +// IndexStatsIndexStoreResponse defines index stats index store information structure +type IndexStatsIndexStoreResponse struct { + SizeInBytes int64 `json:"size_in_bytes"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` +} + +// IndexStatsIndexIndexingResponse defines index stats index indexing information structure +type IndexStatsIndexIndexingResponse struct { + IndexTotal int64 `json:"index_total"` + IndexTimeInMillis int64 `json:"index_time_in_millis"` + IndexCurrent int64 `json:"index_current"` + IndexFailed int64 `json:"index_failed"` + DeleteTotal int64 `json:"delete_total"` + DeleteTimeInMillis int64 `json:"delete_time_in_millis"` + DeleteCurrent int64 `json:"delete_current"` + NoopUpdateTotal int64 `json:"noop_update_total"` + IsThrottled bool `json:"is_throttled"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` +} + +// IndexStatsIndexGetResponse defines index stats index get information structure +type IndexStatsIndexGetResponse struct { + Total int64 `json:"total"` + TimeInMillis int64 `json:"time_in_millis"` + ExistsTotal int64 `json:"exists_total"` + ExistsTimeInMillis int64 `json:"exists_time_in_millis"` + MissingTotal int64 `json:"missing_total"` + MissingTimeInMillis int64 `json:"missing_time_in_millis"` + Current int64 `json:"current"` +} + +// IndexStatsIndexSearchResponse defines index stats index search information structure +type IndexStatsIndexSearchResponse struct { + OpenContexts int64 `json:"open_contexts"` + QueryTotal int64 `json:"query_total"` + QueryTimeInMillis int64 `json:"query_time_in_millis"` + QueryCurrent int64 `json:"query_current"` + FetchTotal int64 `json:"fetch_total"` + FetchTimeInMillis int64 `json:"fetch_time_in_millis"` + FetchCurrent int64 `json:"fetch_current"` + ScrollTotal int64 `json:"scroll_total"` + ScrollTimeInMillis int64 `json:"scroll_time_in_millis"` + ScrollCurrent int64 `json:"scroll_current"` + SuggestTotal int64 `json:"suggest_total"` + SuggestTimeInMillis int64 `json:"suggest_time_in_millis"` + SuggestCurrent int64 `json:"suggest_current"` +} + +// IndexStatsIndexMergesResponse defines index stats index merges information structure +type IndexStatsIndexMergesResponse struct { + Current int64 `json:"current"` + CurrentDocs int64 `json:"current_docs"` + CurrentSizeInBytes int64 `json:"current_size_in_bytes"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` + TotalDocs int64 `json:"total_docs"` + TotalSizeInBytes int64 `json:"total_size_in_bytes"` + TotalStoppedTimeInMillis int64 `json:"total_stopped_time_in_millis"` + TotalThrottledTimeInMillis int64 `json:"total_throttled_time_in_millis"` + TotalAutoThrottleInBytes int64 `json:"total_auto_throttle_in_bytes"` +} + +// IndexStatsIndexRefreshResponse defines index stats index refresh information structure +type IndexStatsIndexRefreshResponse struct { + ExternalTotal int64 `json:"external_total"` + ExternalTotalTime int64 `json:"external_total_time_in_millis"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` + Listeners int64 `json:"listeners"` +} + +// IndexStatsIndexFlushResponse defines index stats index flush information structure +type IndexStatsIndexFlushResponse struct { + Periodic int64 `json:"periodic"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` +} + +// IndexStatsIndexWarmerResponse defines index stats index warmer information structure +type IndexStatsIndexWarmerResponse struct { + Current int64 `json:"current"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` +} + +// IndexStatsIndexQueryCacheResponse defines index stats index query cache information structure +type IndexStatsIndexQueryCacheResponse struct { + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + TotalCount int64 `json:"total_count"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` + CacheSize int64 `json:"cache_size"` + CacheCount int64 `json:"cache_count"` + Evictions int64 `json:"evictions"` +} + +// IndexStatsIndexFielddataResponse defines index stats index fielddata information structure +type IndexStatsIndexFielddataResponse struct { + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + Evictions int64 `json:"evictions"` +} + +// IndexStatsIndexCompletionResponse defines index stats index completion information structure +type IndexStatsIndexCompletionResponse struct { + SizeInBytes int64 `json:"size_in_bytes"` +} + +// IndexStatsIndexSegmentsResponse defines index stats index segments information structure +type IndexStatsIndexSegmentsResponse struct { + Count int64 `json:"count"` + MemoryInBytes int64 `json:"memory_in_bytes"` + TermsMemoryInBytes int64 `json:"terms_memory_in_bytes"` + StoredFieldsMemoryInBytes int64 `json:"stored_fields_memory_in_bytes"` + TermVectorsMemoryInBytes int64 `json:"term_vectors_memory_in_bytes"` + NormsMemoryInBytes int64 `json:"norms_memory_in_bytes"` + PointsMemoryInBytes int64 `json:"points_memory_in_bytes"` + DocValuesMemoryInBytes int64 `json:"doc_values_memory_in_bytes"` + IndexWriterMemoryInBytes int64 `json:"index_writer_memory_in_bytes"` + VersionMapMemoryInBytes int64 `json:"version_map_memory_in_bytes"` + FixedBitSetMemoryInBytes int64 `json:"fixed_bit_set_memory_in_bytes"` + MaxUnsafeAutoIDTimestamp int64 `json:"max_unsafe_auto_id_timestamp"` +} + +// IndexStatsIndexTranslogResponse defines index stats index translog information structure +type IndexStatsIndexTranslogResponse struct { + EarliestLastModifiedAge int64 `json:"earliest_last_modified_age"` + Operations int64 `json:"operations"` + SizeInBytes int64 `json:"size_in_bytes"` + UncommittedOperations int64 `json:"uncommitted_operations"` + UncommittedSizeInBytes int64 `json:"uncommitted_size_in_bytes"` +} + +// IndexStatsIndexRequestCacheResponse defines index stats index request cache information structure +type IndexStatsIndexRequestCacheResponse struct { + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + Evictions int64 `json:"evictions"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` +} + +// IndexStatsIndexRecoveryResponse defines index stats index recovery information structure +type IndexStatsIndexRecoveryResponse struct { + CurrentAsSource int64 `json:"current_as_source"` + CurrentAsTarget int64 `json:"current_as_target"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` +} diff --git a/inputs/elasticsearch/collector/indices_settings.go b/inputs/elasticsearch/collector/indices_settings.go new file mode 100644 index 00000000..5fee59ce --- /dev/null +++ b/inputs/elasticsearch/collector/indices_settings.go @@ -0,0 +1,212 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strconv" + + "github.com/prometheus/client_golang/prometheus" +) + +// IndicesSettings information struct +type IndicesSettings struct { + client *http.Client + url *url.URL + + up prometheus.Gauge + readOnlyIndices prometheus.Gauge + + totalScrapes, jsonParseFailures prometheus.Counter + metrics []*indicesSettingsMetric +} + +var ( + defaultIndicesTotalFieldsLabels = []string{"index"} + defaultTotalFieldsValue = 1000 //es default configuration for total fields + defaultDateCreation = 0 //es index default creation date +) + +type indicesSettingsMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(indexSettings Settings) float64 +} + +// NewIndicesSettings defines Indices Settings Prometheus metrics +func NewIndicesSettings(client *http.Client, url *url.URL) *IndicesSettings { + return &IndicesSettings{ + client: client, + url: url, + + up: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "up"), + Help: "Was the last scrape of the Elasticsearch Indices Settings endpoint successful.", + }), + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "total_scrapes"), + Help: "Current total Elasticsearch Indices Settings scrapes.", + }), + readOnlyIndices: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "read_only_indices"), + Help: "Current number of read only indices within cluster", + }), + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + metrics: []*indicesSettingsMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_settings", "total_fields"), + "index mapping setting for total_fields", + defaultIndicesTotalFieldsLabels, nil, + ), + Value: func(indexSettings Settings) float64 { + val, err := strconv.ParseFloat(indexSettings.IndexInfo.Mapping.TotalFields.Limit, 64) + if err != nil { + return float64(defaultTotalFieldsValue) + } + return val + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_settings", "replicas"), + "index setting number_of_replicas", + defaultIndicesTotalFieldsLabels, nil, + ), + Value: func(indexSettings Settings) float64 { + val, err := strconv.ParseFloat(indexSettings.IndexInfo.NumberOfReplicas, 64) + if err != nil { + return float64(defaultTotalFieldsValue) + } + return val + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_settings", "creation_timestamp_seconds"), + "index setting creation_date", + defaultIndicesTotalFieldsLabels, nil, + ), + Value: func(indexSettings Settings) float64 { + val, err := strconv.ParseFloat(indexSettings.IndexInfo.CreationDate, 64) + if err != nil { + return float64(defaultDateCreation) + } + return val / 1000.0 + }, + }, + }, + } +} + +// Describe add Snapshots metrics descriptions +func (cs *IndicesSettings) Describe(ch chan<- *prometheus.Desc) { + ch <- cs.up.Desc() + ch <- cs.totalScrapes.Desc() + ch <- cs.readOnlyIndices.Desc() + ch <- cs.jsonParseFailures.Desc() +} + +func (cs *IndicesSettings) getAndParseURL(u *url.URL, data interface{}) error { + res, err := cs.client.Get(u.String()) + if err != nil { + return fmt.Errorf("failed to get from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err :", err) + } + }() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + cs.jsonParseFailures.Inc() + return err + } + + if err := json.Unmarshal(bts, data); err != nil { + cs.jsonParseFailures.Inc() + return err + } + return nil +} + +func (cs *IndicesSettings) fetchAndDecodeIndicesSettings() (IndicesSettingsResponse, error) { + + u := *cs.url + u.Path = path.Join(u.Path, "/_all/_settings") + var asr IndicesSettingsResponse + err := cs.getAndParseURL(&u, &asr) + if err != nil { + return asr, err + } + + return asr, err +} + +// Collect gets all indices settings metric values +func (cs *IndicesSettings) Collect(ch chan<- prometheus.Metric) { + + cs.totalScrapes.Inc() + defer func() { + ch <- cs.up + ch <- cs.totalScrapes + ch <- cs.jsonParseFailures + ch <- cs.readOnlyIndices + }() + + asr, err := cs.fetchAndDecodeIndicesSettings() + if err != nil { + cs.readOnlyIndices.Set(0) + cs.up.Set(0) + log.Println("failed to fetch and decode cluster settings stats, err :", err) + return + } + cs.up.Set(1) + + var c int + for indexName, value := range asr { + if value.Settings.IndexInfo.Blocks.ReadOnly == "true" { + c++ + } + for _, metric := range cs.metrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(value.Settings), + indexName, + ) + } + } + cs.readOnlyIndices.Set(float64(c)) +} diff --git a/inputs/elasticsearch/collector/indices_settings_response.go b/inputs/elasticsearch/collector/indices_settings_response.go new file mode 100644 index 00000000..cac3ee0b --- /dev/null +++ b/inputs/elasticsearch/collector/indices_settings_response.go @@ -0,0 +1,50 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +// IndicesSettingsResponse is a representation of Elasticsearch Settings for each Index +type IndicesSettingsResponse map[string]Index + +// Index defines the struct of the tree for the settings of each index +type Index struct { + Settings Settings `json:"settings"` +} + +// Settings defines current index settings +type Settings struct { + IndexInfo IndexInfo `json:"index"` +} + +// IndexInfo defines the blocks of the current index +type IndexInfo struct { + Blocks Blocks `json:"blocks"` + Mapping Mapping `json:"mapping"` + NumberOfReplicas string `json:"number_of_replicas"` + CreationDate string `json:"creation_date"` +} + +// Blocks defines whether current index has read_only_allow_delete enabled +type Blocks struct { + ReadOnly string `json:"read_only_allow_delete"` +} + +// Mapping defines mapping settings +type Mapping struct { + TotalFields TotalFields `json:"total_fields"` +} + +// TotalFields defines the limit on the number of mapped fields +type TotalFields struct { + Limit string `json:"limit"` +} diff --git a/inputs/elasticsearch/collector/indices_settings_test.go b/inputs/elasticsearch/collector/indices_settings_test.go new file mode 100644 index 00000000..fe625ac4 --- /dev/null +++ b/inputs/elasticsearch/collector/indices_settings_test.go @@ -0,0 +1,104 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestIndicesSettings(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION + // curl -XPUT http://localhost:9200/twitter + // curl -XPUT http://localhost:9200/facebook + // curl -XPUT http://localhost:9200/instagram + // curl -XPUT http://localhost:9200/viber + // curl -XPUT http://localhost:9200/instagram/_settings --header "Content-Type: application/json" -d ' + // { + // "index": { + // "mapping": { + // "total_fields": { + // "limit": 10000 + // } + // }, + // "blocks": { + // "read_only_allow_delete": "true" + // } + // } + // }' + // curl -XPUT http://localhost:9200/twitter/_settings --header "Content-Type: application/json" -d ' + // { + // "index": { + // "blocks": { + // "read_only_allow_delete": "true" + // } + // } + // }' + + // curl http://localhost:9200/_all/_settings + + tcs := map[string]string{ + "6.5.4": `{"viber":{"settings":{"index":{"creation_date":"1618593207186","number_of_shards":"5","number_of_replicas":"1","uuid":"lWg86KTARzO3r7lELytT1Q","version":{"created":"6050499"},"provided_name":"viber"}}},"instagram":{"settings":{"index":{"mapping":{"total_fields":{"limit":"10000"}},"number_of_shards":"5","blocks":{"read_only_allow_delete":"true"},"provided_name":"instagram","creation_date":"1618593203353","number_of_replicas":"1","uuid":"msb6eG7aT8GmNe-a4oyVtQ","version":{"created":"6050499"}}}},"twitter":{"settings":{"index":{"number_of_shards":"5","blocks":{"read_only_allow_delete":"true"},"provided_name":"twitter","creation_date":"1618593193641","number_of_replicas":"1","uuid":"YRUT8t4aSkKsNmGl7K3y4Q","version":{"created":"6050499"}}}},"facebook":{"settings":{"index":{"creation_date":"1618593199101","number_of_shards":"5","number_of_replicas":"1","uuid":"trZhb_YOTV-RWKitTYw81A","version":{"created":"6050499"},"provided_name":"facebook"}}}}`, + } + for ver, out := range tcs { + for hn, handler := range map[string]http.Handler{ + "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + }), + } { + ts := httptest.NewServer(handler) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + c := NewIndicesSettings(http.DefaultClient, u) + nsr, err := c.fetchAndDecodeIndicesSettings() + if err != nil { + t.Fatalf("Failed to fetch or decode indices settings: %s", err) + } + t.Logf("[%s/%s] All Indices Settings Response: %+v", hn, ver, nsr) + // if nsr.Cluster.Routing.Allocation.Enabled != "ALL" { + // t.Errorf("Wrong setting for cluster routing allocation enabled") + // } + var counter int + var totalFields int + for key, value := range nsr { + if value.Settings.IndexInfo.Blocks.ReadOnly == "true" { + counter++ + if key != "instagram" && key != "twitter" { + t.Errorf("Wrong read_only index") + } + } + if value.Settings.IndexInfo.Mapping.TotalFields.Limit == "10000" { + totalFields++ + if key != "instagram" { + t.Errorf("Expected 10000 total_fields only for instagram") + } + } + } + if counter != 2 { + t.Errorf("Wrong number of read_only indexes") + } + if totalFields != 1 { + t.Errorf(("Wrong number of total_fields found")) + } + } + } +} diff --git a/inputs/elasticsearch/collector/indices_test.go b/inputs/elasticsearch/collector/indices_test.go new file mode 100644 index 00000000..9244d222 --- /dev/null +++ b/inputs/elasticsearch/collector/indices_test.go @@ -0,0 +1,129 @@ +package collector + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" +) + +func TestIndices(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/foo_1/type1/1 -d '{"title":"abc","content":"hello"}' + // curl -XPUT http://localhost:9200/foo_1/type1/2 -d '{"title":"def","content":"world"}' + // curl -XPUT http://localhost:9200/foo_2/type1/1 -d '{"title":"abc001","content":"hello001"}' + // curl -XPUT http://localhost:9200/foo_2/type1/2 -d '{"title":"def002","content":"world002"}' + // curl -XPUT http://localhost:9200/foo_2/type1/3 -d '{"title":"def003","content":"world003"}' + // curl http://localhost:9200/_all/_stats + ti := map[string]string{ + "1.7.6": `{"_shards":{"total":20,"successful":10,"failed":0},"_all":{"primaries":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":13798,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":52,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":5,"total_time_in_millis":163},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":18410,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":671088640,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":13798,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":52,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":5,"total_time_in_millis":163},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":18410,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":671088640,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":8207,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":38},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":8207,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":38},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":5591,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":2,"total_time_in_millis":125},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":7364,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":5591,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":2,"total_time_in_millis":125},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":7364,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + "2.4.5": `{"_shards":{"total":20,"successful":10,"failed":0},"_all":{"primaries":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":3610,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":40,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":171},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":10530,"terms_memory_in_bytes":7550,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103887660,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":843},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":3610,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":40,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":171},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":10530,"terms_memory_in_bytes":7550,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103887660,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":843},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":3350,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":34},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":470},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":3350,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":34},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":470},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":34,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":137},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4212,"terms_memory_in_bytes":3020,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":373},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":34,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":137},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4212,"terms_memory_in_bytes":3020,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":373},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + "5.4.2": `{"_shards":{"total":26,"successful":13,"failed":0},"_all":{"primaries":{"docs":{"count":76,"deleted":0},"store":{"size_in_bytes":128534,"throttle_time_in_millis":0},"indexing":{"index_total":78,"index_time_in_millis":1598,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":272629760},"refresh":{"total":15,"total_time_in_millis":1361,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":26,"total_time_in_millis":124},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":13,"memory_in_bytes":56523,"terms_memory_in_bytes":44419,"stored_fields_memory_in_bytes":4056,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":2880,"points_memory_in_bytes":652,"doc_values_memory_in_bytes":4516,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":78,"size_in_bytes":56679},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":76,"deleted":0},"store":{"size_in_bytes":128534,"throttle_time_in_millis":0},"indexing":{"index_total":78,"index_time_in_millis":1598,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":272629760},"refresh":{"total":15,"total_time_in_millis":1361,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":26,"total_time_in_millis":124},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":13,"memory_in_bytes":56523,"terms_memory_in_bytes":44419,"stored_fields_memory_in_bytes":4056,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":2880,"points_memory_in_bytes":652,"doc_values_memory_in_bytes":4516,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":78,"size_in_bytes":56679},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-es-2-2017.08.23":{"primaries":{"docs":{"count":65,"deleted":0},"store":{"size_in_bytes":68917,"throttle_time_in_millis":0},"indexing":{"index_total":65,"index_time_in_millis":106,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":390,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":15},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":23830,"terms_memory_in_bytes":18474,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":320,"points_memory_in_bytes":648,"doc_values_memory_in_bytes":3452,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":65,"size_in_bytes":37990},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":65,"deleted":0},"store":{"size_in_bytes":68917,"throttle_time_in_millis":0},"indexing":{"index_total":65,"index_time_in_millis":106,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":390,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":15},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":23830,"terms_memory_in_bytes":18474,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":320,"points_memory_in_bytes":648,"doc_values_memory_in_bytes":3452,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":65,"size_in_bytes":37990},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-data-2":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":4226,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":74,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":3,"total_time_in_millis":2},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1335,"terms_memory_in_bytes":787,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":236,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":6738},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":4226,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":74,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":3,"total_time_in_millis":2},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1335,"terms_memory_in_bytes":787,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":236,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":6738},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11909,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":12,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":42,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":4},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":494},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11909,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":12,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":42,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":4},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":494},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8038,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":84,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":7,"total_time_in_millis":94},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":2,"size_in_bytes":389},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8038,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":84,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":7,"total_time_in_millis":94},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":2,"size_in_bytes":389},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".watches":{"primaries":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":35444,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":1421,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":5,"total_time_in_millis":771,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":9},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":18418,"terms_memory_in_bytes":15198,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":4,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":11068},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":35444,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":1421,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":5,"total_time_in_millis":771,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":9},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":18418,"terms_memory_in_bytes":15198,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":4,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":11068},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + "5.5.2": `{"_shards":{"total":34,"successful":34,"failed":0},"_all":{"primaries":{"docs":{"count":256658,"deleted":148},"store":{"size_in_bytes":175959144,"throttle_time_in_millis":0},"indexing":{"index_total":413652,"index_time_in_millis":316231,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88458,"query_time_in_millis":2513,"query_current":0,"fetch_total":88446,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":10275,"total_time_in_millis":5895937,"total_docs":80376844,"total_size_in_bytes":49005089428,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":354609338},"refresh":{"total":100264,"total_time_in_millis":818781,"listeners":0},"flush":{"total":5,"total_time_in_millis":124},"warmer":{"current":0,"total":100286,"total_time_in_millis":23851},"query_cache":{"memory_size_in_bytes":0,"total_count":22,"hit_count":0,"miss_count":22,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":38,"memory_in_bytes":720166,"terms_memory_in_bytes":481188,"stored_fields_memory_in_bytes":31920,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":3072,"points_memory_in_bytes":61858,"doc_values_memory_in_bytes":142128,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":137331,"size_in_bytes":167642357},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":513316,"deleted":324},"store":{"size_in_bytes":352143858,"throttle_time_in_millis":0},"indexing":{"index_total":827302,"index_time_in_millis":625030,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176913,"query_time_in_millis":5115,"query_current":0,"fetch_total":176891,"fetch_time_in_millis":5862,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":20547,"total_time_in_millis":11765629,"total_docs":160766745,"total_size_in_bytes":98024993696,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":709218676},"refresh":{"total":200378,"total_time_in_millis":1620051,"listeners":0},"flush":{"total":10,"total_time_in_millis":264},"warmer":{"current":0,"total":200427,"total_time_in_millis":46689},"query_cache":{"memory_size_in_bytes":0,"total_count":54,"hit_count":0,"miss_count":54,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":776,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":86,"memory_in_bytes":1496932,"terms_memory_in_bytes":1008932,"stored_fields_memory_in_bytes":66896,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":6784,"points_memory_in_bytes":124368,"doc_values_memory_in_bytes":289952,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":274662,"size_in_bytes":335284714},"request_cache":{"memory_size_in_bytes":1794,"evictions":0,"hit_count":0,"miss_count":3},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-kibana-6-2017.08.16":{"primaries":{"docs":{"count":12459,"deleted":0},"store":{"size_in_bytes":4880578,"throttle_time_in_millis":0},"indexing":{"index_total":12459,"index_time_in_millis":20073,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1372,"total_time_in_millis":275560,"total_docs":8619963,"total_size_in_bytes":3518738895,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":12724,"total_time_in_millis":79600,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12725,"total_time_in_millis":2653},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":34337,"terms_memory_in_bytes":30104,"stored_fields_memory_in_bytes":968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":1393,"doc_values_memory_in_bytes":1744,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":12459,"size_in_bytes":12175093},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":24918,"deleted":0},"store":{"size_in_bytes":9682148,"throttle_time_in_millis":0},"indexing":{"index_total":24918,"index_time_in_millis":37852,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2743,"total_time_in_millis":549689,"total_docs":17234974,"total_size_in_bytes":7034624631,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":25390,"total_time_in_millis":155546,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":25392,"total_time_in_millis":5196},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83565,"terms_memory_in_bytes":73488,"stored_fields_memory_in_bytes":2968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":2897,"doc_values_memory_in_bytes":3764,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":24918,"size_in_bytes":24350186},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.15":{"primaries":{"docs":{"count":25917,"deleted":0},"store":{"size_in_bytes":9617967,"throttle_time_in_millis":0},"indexing":{"index_total":25917,"index_time_in_millis":36304,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2055,"total_time_in_millis":746516,"total_docs":27797269,"total_size_in_bytes":10672726928,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":19569,"total_time_in_millis":123083,"listeners":0},"flush":{"total":1,"total_time_in_millis":18},"warmer":{"current":0,"total":19571,"total_time_in_millis":3995},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":58226,"terms_memory_in_bytes":52997,"stored_fields_memory_in_bytes":1912,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":2721,"doc_values_memory_in_bytes":404,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":51834,"deleted":0},"store":{"size_in_bytes":19309686,"throttle_time_in_millis":0},"indexing":{"index_total":51834,"index_time_in_millis":68258,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":4109,"total_time_in_millis":1477106,"total_docs":55598675,"total_size_in_bytes":21347372840,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":39103,"total_time_in_millis":238886,"listeners":0},"flush":{"total":2,"total_time_in_millis":44},"warmer":{"current":0,"total":39108,"total_time_in_millis":7775},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":141204,"terms_memory_in_bytes":128132,"stored_fields_memory_in_bytes":5472,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":5628,"doc_values_memory_in_bytes":1268,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.14":{"primaries":{"docs":{"count":6147,"deleted":0},"store":{"size_in_bytes":2467088,"throttle_time_in_millis":0},"indexing":{"index_total":6147,"index_time_in_millis":9228,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":397,"total_time_in_millis":48142,"total_docs":1231276,"total_size_in_bytes":559560922,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3733,"total_time_in_millis":28841,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":3735,"total_time_in_millis":1168},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":29099,"terms_memory_in_bytes":25724,"stored_fields_memory_in_bytes":1104,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":715,"doc_values_memory_in_bytes":1364,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":12294,"deleted":0},"store":{"size_in_bytes":5029190,"throttle_time_in_millis":0},"indexing":{"index_total":12294,"index_time_in_millis":16557,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":792,"total_time_in_millis":96523,"total_docs":2467549,"total_size_in_bytes":1121117660,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":7449,"total_time_in_millis":51544,"listeners":0},"flush":{"total":2,"total_time_in_millis":16},"warmer":{"current":0,"total":7453,"total_time_in_millis":1872},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":83924,"terms_memory_in_bytes":73590,"stored_fields_memory_in_bytes":3768,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":1618,"doc_values_memory_in_bytes":4244,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".kibana":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3967,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88441,"query_time_in_millis":2287,"query_current":0,"fetch_total":88441,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":19,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":5,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2219,"terms_memory_in_bytes":1751,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":64,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":7934,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176879,"query_time_in_millis":4703,"query_current":0,"fetch_total":176879,"fetch_time_in_millis":5860,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":4,"total_time_in_millis":33,"listeners":0},"flush":{"total":2,"total_time_in_millis":14},"warmer":{"current":0,"total":10,"total_time_in_millis":26},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4438,"terms_memory_in_bytes":3502,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.15":{"primaries":{"docs":{"count":120970,"deleted":60},"store":{"size_in_bytes":90471073,"throttle_time_in_millis":0},"indexing":{"index_total":207360,"index_time_in_millis":142584,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":85,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3781,"total_time_in_millis":2188959,"total_docs":19326607,"total_size_in_bytes":15981823826,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":19065018},"refresh":{"total":37364,"total_time_in_millis":337011,"listeners":0},"flush":{"total":1,"total_time_in_millis":67},"warmer":{"current":0,"total":37365,"total_time_in_millis":8943},"query_cache":{"memory_size_in_bytes":0,"total_count":9,"hit_count":0,"miss_count":9,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":297633,"terms_memory_in_bytes":184606,"stored_fields_memory_in_bytes":13736,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":31279,"doc_values_memory_in_bytes":67436,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":241940,"deleted":120},"store":{"size_in_bytes":180988889,"throttle_time_in_millis":0},"indexing":{"index_total":414720,"index_time_in_millis":286230,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":135,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":7561,"total_time_in_millis":4425176,"total_docs":38722769,"total_size_in_bytes":32018268991,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":38130036},"refresh":{"total":74706,"total_time_in_millis":675900,"listeners":0},"flush":{"total":2,"total_time_in_millis":148},"warmer":{"current":0,"total":74710,"total_time_in_millis":18069},"query_cache":{"memory_size_in_bytes":0,"total_count":24,"hit_count":0,"miss_count":24,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":424,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":16,"memory_in_bytes":590531,"terms_memory_in_bytes":362031,"stored_fields_memory_in_bytes":26728,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1024,"points_memory_in_bytes":62748,"doc_values_memory_in_bytes":138000,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755201903,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.14":{"primaries":{"docs":{"count":24614,"deleted":12},"store":{"size_in_bytes":17680221,"throttle_time_in_millis":0},"indexing":{"index_total":36896,"index_time_in_millis":35441,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":24,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":927,"total_time_in_millis":1245791,"total_docs":11392506,"total_size_in_bytes":8435013687,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":9432,"total_time_in_millis":90529,"listeners":0},"flush":{"total":1,"total_time_in_millis":25},"warmer":{"current":0,"total":9434,"total_time_in_millis":3098},"query_cache":{"memory_size_in_bytes":0,"total_count":2,"hit_count":0,"miss_count":2,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83467,"terms_memory_in_bytes":59601,"stored_fields_memory_in_bytes":4040,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":7054,"doc_values_memory_in_bytes":12324,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":49228,"deleted":18},"store":{"size_in_bytes":35172184,"throttle_time_in_millis":0},"indexing":{"index_total":73792,"index_time_in_millis":69939,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":61,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1853,"total_time_in_millis":2468444,"total_docs":22769560,"total_size_in_bytes":16842830754,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":18843,"total_time_in_millis":176757,"listeners":0},"flush":{"total":2,"total_time_in_millis":42},"warmer":{"current":0,"total":18847,"total_time_in_millis":5819},"query_cache":{"memory_size_in_bytes":0,"total_count":7,"hit_count":0,"miss_count":7,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":352,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":137869,"terms_memory_in_bytes":97131,"stored_fields_memory_in_bytes":6680,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":13662,"doc_values_memory_in_bytes":19820,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":12138,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":5,"query_time_in_millis":0,"query_current":0,"fetch_total":2,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":12,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":524},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":6,"deleted":0},"store":{"size_in_bytes":24276,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10,"query_time_in_millis":0,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":20,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":6,"memory_in_bytes":15528,"terms_memory_in_bytes":11952,"stored_fields_memory_in_bytes":1872,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1152,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":552,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":6,"size_in_bytes":1048},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8246,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":7,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":7,"query_time_in_millis":8,"query_current":0,"fetch_total":3,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":4,"total_time_in_millis":18,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":9,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":609},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":16492,"throttle_time_in_millis":0},"indexing":{"index_total":7,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":15,"query_time_in_millis":21,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":7,"total_time_in_millis":32,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":18,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":10352,"terms_memory_in_bytes":7968,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":768,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":8,"size_in_bytes":1218},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.16":{"primaries":{"docs":{"count":66545,"deleted":76},"store":{"size_in_bytes":50817866,"throttle_time_in_millis":0},"indexing":{"index_total":124865,"index_time_in_millis":72588,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":67,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1743,"total_time_in_millis":1390969,"total_docs":12009223,"total_size_in_bytes":9837225170,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":17433,"total_time_in_millis":159668,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":17434,"total_time_in_millis":3982},"query_cache":{"memory_size_in_bytes":0,"total_count":8,"hit_count":0,"miss_count":8,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":8,"memory_in_bytes":202245,"terms_memory_in_bytes":116445,"stored_fields_memory_in_bytes":8288,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":512,"points_memory_in_bytes":18696,"doc_values_memory_in_bytes":58304,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":124865,"size_in_bytes":155465916},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":133090,"deleted":186},"store":{"size_in_bytes":101913059,"throttle_time_in_millis":0},"indexing":{"index_total":249730,"index_time_in_millis":146171,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":143,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3489,"total_time_in_millis":2748691,"total_docs":23973218,"total_size_in_bytes":19660778820,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":34871,"total_time_in_millis":321333,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":34873,"total_time_in_millis":7932},"query_cache":{"memory_size_in_bytes":0,"total_count":17,"hit_count":0,"miss_count":17,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":20,"memory_in_bytes":429521,"terms_memory_in_bytes":251138,"stored_fields_memory_in_bytes":17536,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1280,"points_memory_in_bytes":37815,"doc_values_memory_in_bytes":121752,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":249730,"size_in_bytes":310931832},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + } + for ver, out := range ti { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + indices := make([]string, 0) + i := NewIndices(http.DefaultClient, u, false, false, indices) + stats, err := i.fetchAndDecodeIndexStats() + if err != nil { + t.Fatalf("Failed to fetch or decode indices stats: %s", err) + } + t.Logf("[%s] Index Response: %+v", ver, stats) + if stats.Indices["foo_1"].Primaries.Docs.Count != 2 { + t.Errorf("Wrong number of primary docs") + } + if stats.Indices["foo_1"].Primaries.Store.SizeInBytes == 0 { + t.Errorf("Wrong number of primary store size in bytes") + } + if stats.Indices["foo_1"].Total.Store.SizeInBytes == 0 { + t.Errorf("Wrong number of total store size in bytes") + } + if stats.Indices["foo_1"].Total.Indexing.IndexTimeInMillis == 0 { + t.Errorf("Wrong indexing time recorded") + } + if stats.Indices["foo_1"].Total.Indexing.IndexTotal == 0 { + t.Errorf("Wrong indexing total recorded") + } + if stats.Aliases != nil { + t.Errorf("Aliases are populated when they should not be") + } + } +} + +func TestAliases(t *testing.T) { + /* + Testcases created using (note: the docker image name no longer has -alpine in it for newer versions): + docker run -d -p 9200:9200 elasticsearch:VERSION-alpine # versions 1.x.x, 2.x.x, and 5.x.x + docker run -p 9200:9200 -e "discovery.type=single-node" elasticsearch:VERSION # version 7.x.x + curl -XPUT -H "Content-Type: application/json" http://localhost:9200/foo_1/type1/1 -d '{"title":"abc","content":"hello"}' + curl -XPUT -H "Content-Type: application/json" http://localhost:9200/foo_2/type1/1 -d '{"title":"abc001","content":"hello001"}' + curl -XPUT -H "Content-Type: application/json" http://localhost:9200/foo_3/type1/1 -d '{"title":"def003","content":"world003"}' + curl -XPOST -H "Content-Type: application/json" http://localhost:9200/_aliases -d '{"actions": [{"add": {"index": "foo_2","alias": "foo_alias_2_1"}}]}' + curl -XPOST -H "Content-Type: application/json" http://localhost:9200/_aliases -d '{"actions": [{"add": {"index": "foo_3","alias": "foo_alias_3_2"}}]}' + curl -XPOST -H "Content-Type: application/json" http://localhost:9200/_aliases -d '{"actions": [{"add": {"index": "foo_3","alias": "foo_alias_3_1", "is_write_index": true, "routing": "title"}}]}' + curl -H "Content-Type: application/json" http://localhost:9200/_alias + curl -H "Content-Type: application/json" http://localhost:9200/_all/_stats + */ + ti := map[string]map[string]string{ + "1.7.6": { + "alias": `{"foo_1":{"aliases":{}},"foo_2":{"aliases":{"foo_alias_2_1":{}}},"foo_3":{"aliases":{"foo_alias_3_1":{"index_routing":"title","search_routing":"title","is_write_index":true},"foo_alias_3_2":{}}}}`, + "stats": `{"_shards":{"total":30,"successful":15,"failed":0},"_all":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":9336,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":69,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":129},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":36,"total_time_in_millis":20},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":1006632960,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":9336,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":69,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":129},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":36,"total_time_in_millis":20},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":1006632960,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3124,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":1,"total_time_in_millis":14},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":3682,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3124,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":1,"total_time_in_millis":14},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":3682,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3088,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":65,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":1,"total_time_in_millis":101},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":20},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":3682,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3088,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":65,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":1,"total_time_in_millis":101},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":20},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":3682,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_3":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3124,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":1,"total_time_in_millis":14},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":3682,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3124,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":1,"total_time_in_millis":14},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":3682,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + }, + "2.4.5": { + "alias": `{"foo_1":{"aliases":{}},"foo_2":{"aliases":{"foo_alias_2_1":{}}},"foo_3":{"aliases":{"foo_alias_3_1":{"index_routing":"title","search_routing":"title","is_write_index":true},"foo_alias_3_2":{}}}}`, + "stats": `{"_shards":{"total":30,"successful":15,"failed":0},"_all":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11164,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":59,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":314572800},"refresh":{"total":3,"total_time_in_millis":148},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":36,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103795905,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":894},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11164,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":59,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":314572800},"refresh":{"total":3,"total_time_in_millis":148},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":36,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103795905,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":894},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3733,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":16},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2106,"terms_memory_in_bytes":1510,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":34598635,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":300},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3733,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":16},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2106,"terms_memory_in_bytes":1510,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":34598635,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":300},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3698,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":56,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":116},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2106,"terms_memory_in_bytes":1510,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":34598635,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":294},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3698,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":56,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":116},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2106,"terms_memory_in_bytes":1510,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":34598635,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":294},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_3":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3733,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":16},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2106,"terms_memory_in_bytes":1510,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":34598635,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":300},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3733,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":16},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2106,"terms_memory_in_bytes":1510,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":34598635,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":1,"size_in_bytes":300},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + }, + "5.4.2": { + "alias": `{"foo_1":{"aliases":{}},"foo_2":{"aliases":{"foo_alias_2_1":{}}},"foo_3":{"aliases":{"foo_alias_3_1":{"index_routing":"title","search_routing":"title","is_write_index":true},"foo_alias_3_2":{}}}}`, + "stats": `{"_shards":{"total":30,"successful":15,"failed":0},"_all":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":13165,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":80,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":314572800},"refresh":{"total":3,"total_time_in_millis":1317,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":18,"total_time_in_millis":26},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":918},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":13165,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":80,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":314572800},"refresh":{"total":3,"total_time_in_millis":1317,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":18,"total_time_in_millis":26},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":918},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":4408,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":24,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":308},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":4408,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":24,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":308},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":4349,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":3,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":1166,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":6},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":302},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":4349,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":3,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":1166,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":6},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":302},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_3":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":4408,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":73,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":127,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":20},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":308},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":4408,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":73,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":127,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":20},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":308},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + }, + "7.17.3": { + "alias": `{"foo_1":{"aliases":{}},"foo_2":{"aliases":{"foo_alias_2_1":{}}},"foo_3":{"aliases":{"foo_alias_3_1":{"index_routing":"title","search_routing":"title","is_write_index":true},"foo_alias_3_2":{}}}}`, + "stats": `{"_shards":{"total":7,"successful":4,"failed":0},"_all":{"primaries":{"docs":{"count":43,"deleted":0},"shard_stats":{"total_count":4},"store":{"size_in_bytes":39917364,"total_data_set_size_in_bytes":39917364,"reserved_in_bytes":0},"indexing":{"index_total":43,"index_time_in_millis":741,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":43,"query_time_in_millis":71,"query_current":0,"fetch_total":43,"fetch_time_in_millis":96,"fetch_current":0,"scroll_total":3,"scroll_time_in_millis":60,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":83886080},"refresh":{"total":18,"total_time_in_millis":76,"external_total":15,"external_total_time_in_millis":73,"listeners":0},"flush":{"total":4,"periodic":0,"total_time_in_millis":150},"warmer":{"current":0,"total":11,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":9996,"terms_memory_in_bytes":5600,"stored_fields_memory_in_bytes":3480,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":532,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":487,"uncommitted_operations":3,"uncommitted_size_in_bytes":487,"earliest_last_modified_age":35855},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":43,"deleted":0},"shard_stats":{"total_count":4},"store":{"size_in_bytes":39917364,"total_data_set_size_in_bytes":39917364,"reserved_in_bytes":0},"indexing":{"index_total":43,"index_time_in_millis":741,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":43,"query_time_in_millis":71,"query_current":0,"fetch_total":43,"fetch_time_in_millis":96,"fetch_current":0,"scroll_total":3,"scroll_time_in_millis":60,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":83886080},"refresh":{"total":18,"total_time_in_millis":76,"external_total":15,"external_total_time_in_millis":73,"listeners":0},"flush":{"total":4,"periodic":0,"total_time_in_millis":150},"warmer":{"current":0,"total":11,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":9996,"terms_memory_in_bytes":5600,"stored_fields_memory_in_bytes":3480,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":532,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":487,"uncommitted_operations":3,"uncommitted_size_in_bytes":487,"earliest_last_modified_age":35855},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".geoip_databases":{"uuid":"AbBfA8RRRLGfbIPGIAQs7A","primaries":{"docs":{"count":40,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":39904033,"total_data_set_size_in_bytes":39904033,"reserved_in_bytes":0},"indexing":{"index_total":40,"index_time_in_millis":738,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":43,"query_time_in_millis":71,"query_current":0,"fetch_total":43,"fetch_time_in_millis":96,"fetch_current":0,"scroll_total":3,"scroll_time_in_millis":60,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":9,"total_time_in_millis":50,"external_total":6,"external_total_time_in_millis":45,"listeners":0},"flush":{"total":4,"periodic":0,"total_time_in_millis":150},"warmer":{"current":0,"total":5,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":4368,"terms_memory_in_bytes":2048,"stored_fields_memory_in_bytes":2016,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":304,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":55,"uncommitted_operations":0,"uncommitted_size_in_bytes":55,"earliest_last_modified_age":406186},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":40,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":39904033,"total_data_set_size_in_bytes":39904033,"reserved_in_bytes":0},"indexing":{"index_total":40,"index_time_in_millis":738,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":43,"query_time_in_millis":71,"query_current":0,"fetch_total":43,"fetch_time_in_millis":96,"fetch_current":0,"scroll_total":3,"scroll_time_in_millis":60,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":9,"total_time_in_millis":50,"external_total":6,"external_total_time_in_millis":45,"listeners":0},"flush":{"total":4,"periodic":0,"total_time_in_millis":150},"warmer":{"current":0,"total":5,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":4368,"terms_memory_in_bytes":2048,"stored_fields_memory_in_bytes":2016,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":304,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":55,"uncommitted_operations":0,"uncommitted_size_in_bytes":55,"earliest_last_modified_age":406186},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"uuid":"JnYmxu4DStKroXSy6BHYcQ","primaries":{"docs":{"count":1,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":4459,"total_data_set_size_in_bytes":4459,"reserved_in_bytes":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":9,"external_total":3,"external_total_time_in_millis":10,"listeners":0},"flush":{"total":0,"periodic":0,"total_time_in_millis":0},"warmer":{"current":0,"total":2,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1876,"terms_memory_in_bytes":1184,"stored_fields_memory_in_bytes":488,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":76,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":146,"uncommitted_operations":1,"uncommitted_size_in_bytes":146,"earliest_last_modified_age":36079},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":4459,"total_data_set_size_in_bytes":4459,"reserved_in_bytes":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":9,"external_total":3,"external_total_time_in_millis":10,"listeners":0},"flush":{"total":0,"periodic":0,"total_time_in_millis":0},"warmer":{"current":0,"total":2,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1876,"terms_memory_in_bytes":1184,"stored_fields_memory_in_bytes":488,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":76,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":146,"uncommitted_operations":1,"uncommitted_size_in_bytes":146,"earliest_last_modified_age":36079},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"uuid":"9itiRXMuQym8eTdKygV3Kw","primaries":{"docs":{"count":1,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":4413,"total_data_set_size_in_bytes":4413,"reserved_in_bytes":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":8,"external_total":3,"external_total_time_in_millis":8,"listeners":0},"flush":{"total":0,"periodic":0,"total_time_in_millis":0},"warmer":{"current":0,"total":2,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1876,"terms_memory_in_bytes":1184,"stored_fields_memory_in_bytes":488,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":76,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":140,"uncommitted_operations":1,"uncommitted_size_in_bytes":140,"earliest_last_modified_age":36364},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":4413,"total_data_set_size_in_bytes":4413,"reserved_in_bytes":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":8,"external_total":3,"external_total_time_in_millis":8,"listeners":0},"flush":{"total":0,"periodic":0,"total_time_in_millis":0},"warmer":{"current":0,"total":2,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1876,"terms_memory_in_bytes":1184,"stored_fields_memory_in_bytes":488,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":76,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":140,"uncommitted_operations":1,"uncommitted_size_in_bytes":140,"earliest_last_modified_age":36364},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_3":{"uuid":"a2-lU19tRuKUPgarKpqoCg","primaries":{"docs":{"count":1,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":4459,"total_data_set_size_in_bytes":4459,"reserved_in_bytes":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":9,"external_total":3,"external_total_time_in_millis":10,"listeners":0},"flush":{"total":0,"periodic":0,"total_time_in_millis":0},"warmer":{"current":0,"total":2,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1876,"terms_memory_in_bytes":1184,"stored_fields_memory_in_bytes":488,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":76,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":146,"uncommitted_operations":1,"uncommitted_size_in_bytes":146,"earliest_last_modified_age":35855},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"shard_stats":{"total_count":1},"store":{"size_in_bytes":4459,"total_data_set_size_in_bytes":4459,"reserved_in_bytes":0},"indexing":{"index_total":1,"index_time_in_millis":1,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":9,"external_total":3,"external_total_time_in_millis":10,"listeners":0},"flush":{"total":0,"periodic":0,"total_time_in_millis":0},"warmer":{"current":0,"total":2,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1876,"terms_memory_in_bytes":1184,"stored_fields_memory_in_bytes":488,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":76,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":146,"uncommitted_operations":1,"uncommitted_size_in_bytes":146,"earliest_last_modified_age":35855},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + }, + } + for ver, out := range ti { + mux := http.NewServeMux() + mux.HandleFunc("/_all/_stats", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out["stats"]) + }) + mux.HandleFunc("/_alias", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out["alias"]) + }) + ts := httptest.NewServer(mux) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + indices := make([]string, 0) + i := NewIndices(http.DefaultClient, u, false, true, indices) + stats, err := i.fetchAndDecodeIndexStats() + if err != nil { + t.Fatalf("Failed to fetch or decode indices stats: %s", err) + } + t.Logf("[%s] Index Response: %+v", ver, stats) + if stats.Aliases["foo_1"] != nil { + t.Errorf("Wrong aliases for foo_1") + } + if !reflect.DeepEqual(stats.Aliases["foo_2"], []string{"foo_alias_2_1"}) { + t.Errorf("Wrong aliases for foo_2") + } + if !reflect.DeepEqual(stats.Aliases["foo_3"], []string{"foo_alias_3_1", "foo_alias_3_2"}) { + t.Errorf("Wrong aliases for foo_3") + } + } +} diff --git a/inputs/elasticsearch/collector/nodes.go b/inputs/elasticsearch/collector/nodes.go new file mode 100644 index 00000000..86b3e240 --- /dev/null +++ b/inputs/elasticsearch/collector/nodes.go @@ -0,0 +1,2444 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +func getRoles(node NodeStatsNodeResponse) map[string]bool { + // default settings (2.x) and map, which roles to consider + roles := map[string]bool{ + "master": false, + "data": false, + "data_hot": false, + "data_warm": false, + "data_cold": false, + "data_frozen": false, + "data_content": false, + "ml": false, + "remote_cluster_client": false, + "transform": false, + "ingest": false, + "client": true, + } + // assumption: a 5.x node has at least one role, otherwise it's a 1.7 or 2.x node + if len(node.Roles) > 0 { + for _, role := range node.Roles { + // set every absent role to false + if _, ok := roles[role]; !ok { + roles[role] = false + } else { + // if present in the roles field, set to true + roles[role] = true + } + } + } else { + for role, setting := range node.Attributes { + if _, ok := roles[role]; ok { + if setting == "false" { + roles[role] = false + } else { + roles[role] = true + } + } + } + } + if len(node.HTTP) == 0 { + roles["client"] = false + } + return roles +} + +func createRoleMetric(role string) *nodeMetric { + return &nodeMetric{ + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "nodes", "roles"), + "Node roles", + defaultRoleLabels, prometheus.Labels{"role": role}, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return 1.0 + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return []string{ + cluster, + node.Host, + node.Name, + } + }, + } +} + +var ( + defaultNodeLabels = []string{"cluster", "host", "name", "es_master_node", "es_data_node", "es_ingest_node", "es_client_node"} + defaultRoleLabels = []string{"cluster", "host", "name"} + defaultThreadPoolLabels = append(defaultNodeLabels, "type") + defaultBreakerLabels = append(defaultNodeLabels, "breaker") + defaultFilesystemDataLabels = append(defaultNodeLabels, "mount", "path") + defaultFilesystemIODeviceLabels = append(defaultNodeLabels, "device") + defaultCacheLabels = append(defaultNodeLabels, "cache") + + defaultNodeLabelValues = func(cluster string, node NodeStatsNodeResponse) []string { + roles := getRoles(node) + return []string{ + cluster, + node.Host, + node.Name, + fmt.Sprintf("%t", roles["master"]), + fmt.Sprintf("%t", roles["data"]), + fmt.Sprintf("%t", roles["ingest"]), + fmt.Sprintf("%t", roles["client"]), + } + } + defaultThreadPoolLabelValues = func(cluster string, node NodeStatsNodeResponse, pool string) []string { + return append(defaultNodeLabelValues(cluster, node), pool) + } + defaultFilesystemDataLabelValues = func(cluster string, node NodeStatsNodeResponse, mount string, path string) []string { + return append(defaultNodeLabelValues(cluster, node), mount, path) + } + defaultFilesystemIODeviceLabelValues = func(cluster string, node NodeStatsNodeResponse, device string) []string { + return append(defaultNodeLabelValues(cluster, node), device) + } + defaultCacheHitLabelValues = func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "hit") + } + defaultCacheMissLabelValues = func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "miss") + } +) + +type nodeMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(node NodeStatsNodeResponse) float64 + Labels func(cluster string, node NodeStatsNodeResponse) []string +} + +type gcCollectionMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(gcStats NodeStatsJVMGCCollectorResponse) float64 + Labels func(cluster string, node NodeStatsNodeResponse, collector string) []string +} + +type breakerMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(breakerStats NodeStatsBreakersResponse) float64 + Labels func(cluster string, node NodeStatsNodeResponse, breaker string) []string +} + +type threadPoolMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 + Labels func(cluster string, node NodeStatsNodeResponse, breaker string) []string +} + +type filesystemDataMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(fsStats NodeStatsFSDataResponse) float64 + Labels func(cluster string, node NodeStatsNodeResponse, mount string, path string) []string +} + +type filesystemIODeviceMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(fsStats NodeStatsFSIOStatsDeviceResponse) float64 + Labels func(cluster string, node NodeStatsNodeResponse, device string) []string +} + +// Nodes information struct +type Nodes struct { + client *http.Client + url *url.URL + all bool + node string + local bool + nodeStats []string + + up prometheus.Gauge + totalScrapes, jsonParseFailures prometheus.Counter + + jvmMetrics []*nodeMetric + gcCollectionMetrics []*gcCollectionMetric + osMetrics []*nodeMetric + processMetrics []*nodeMetric + breakerMetrics []*breakerMetric + indicesMetrics []*nodeMetric + transportMetrics []*nodeMetric + threadPoolMetrics []*threadPoolMetric + filesystemDataMetrics []*filesystemDataMetric + filesystemIODeviceMetrics []*filesystemIODeviceMetric +} + +// NewNodes defines Nodes Prometheus metrics +func NewNodes(client *http.Client, url *url.URL, all bool, node string, local bool, nodeStats []string) *Nodes { + return &Nodes{ + client: client, + url: url, + all: all, + node: node, + local: local, + nodeStats: nodeStats, + + up: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "node_stats", "up"), + Help: "Was the last scrape of the Elasticsearch nodes endpoint successful.", + }), + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "node_stats", "total_scrapes"), + Help: "Current total Elasticsearch node scrapes.", + }), + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "node_stats", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + + transportMetrics: []*nodeMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "transport", "rx_packets_total"), + "Count of packets received", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Transport.RxCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "transport", "rx_size_in_bytes_total"), + "Total number of bytes received", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Transport.RxSize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "transport", "tx_packets_total"), + "Count of packets sent", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Transport.TxCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "transport", "tx_size_in_bytes_total"), + "Total number of bytes sent", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Transport.TxSize) + }, + Labels: defaultNodeLabelValues, + }, + }, + jvmMetrics: []*nodeMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm", "threads_count"), + "Count of threads", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Threads.Count) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm", "threads_peak_count"), + "Peak count of threads", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Threads.PeakCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm", "timestamp"), + "JVM timestamp", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Timestamp) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_mem_heap", "used_in_bytes"), + "JVM memory currently used by area", + append(defaultNodeLabels, "area"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.HeapUsed) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "heap") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_mem_non_heap", "used_in_bytes"), + "JVM memory currently used by area", + append(defaultNodeLabels, "area"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.NonHeapUsed) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "non-heap") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_mem_heap", "max_in_bytes"), + "JVM memory max", + append(defaultNodeLabels, "area"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.HeapMax) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "heap") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_mem_heap", "used_percent"), + "JVM memory used percent", + append(defaultNodeLabels, "area"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.HeapUsedPercent) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "heap") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_mem_heap", "committed_in_bytes"), + "JVM memory currently committed by area", + append(defaultNodeLabels, "area"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.HeapCommitted) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "heap") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_mem_non_heap", "committed_in_bytes"), + "JVM memory currently committed by area", + append(defaultNodeLabels, "area"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.NonHeapCommitted) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "non-heap") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_young", "used_in_bytes"), + "JVM memory currently used by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["young"].Used) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "young") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_young", "max_in_bytes"), + "JVM memory max by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["young"].Max) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "young") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_young", "peak_used_in_bytes"), + "JVM memory peak used by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["young"].PeakUsed) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "young") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_young", "peak_max_in_bytes"), + "JVM memory peak max by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["young"].PeakMax) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "young") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_survivor", "used_in_bytes"), + "JVM memory currently used by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["survivor"].Used) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "survivor") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_survivor", "max_in_bytes"), + "JVM memory max by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["survivor"].Max) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "survivor") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_survivor", "peak_used_in_bytes"), + "JVM memory peak used by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["survivor"].PeakUsed) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "survivor") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_survivor", "peak_max_in_bytes"), + "JVM memory peak max by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["survivor"].PeakMax) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "survivor") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_old", "used_in_bytes"), + "JVM memory currently used by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["old"].Used) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "old") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_old", "max_in_bytes"), + "JVM memory max by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["old"].Max) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "old") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_old", "peak_used_in_bytes"), + "JVM memory peak used by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["old"].PeakUsed) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "old") + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_memory_pools_old", "peak_max_in_bytes"), + "JVM memory peak max by pool", + append(defaultNodeLabels, "pool"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Mem.Pools["old"].PeakMax) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "old") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_buffer_pool_direct", "count"), + "JVM buffer count", + append(defaultNodeLabels, "type"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.BufferPools["direct"].Count) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "direct") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_buffer_pool_direct", "total_capacity_in_bytes"), + "JVM buffer total capacity", + append(defaultNodeLabels, "type"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.BufferPools["direct"].TotalCapacity) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "direct") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_buffer_pool_direct", "used_in_bytes"), + "JVM buffer currently used", + append(defaultNodeLabels, "type"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.BufferPools["direct"].Used) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "direct") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_buffer_pool_mapped", "count"), + "JVM buffer count", + append(defaultNodeLabels, "type"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.BufferPools["mapped"].Count) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "mapped") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_buffer_pool_mapped", "total_capacity_in_bytes"), + "JVM buffer total capacity", + append(defaultNodeLabels, "type"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.BufferPools["mapped"].TotalCapacity) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "mapped") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_buffer_pool_mapped", "used_in_bytes"), + "JVM buffer currently used", + append(defaultNodeLabels, "type"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.BufferPools["mapped"].Used) + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "mapped") + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_classes", "current_loaded_count"), + "JVM classes currently loaded", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Classes.CurrentLoadedCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_classes", "total_loaded_count"), + "JVM classes total loaded", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Classes.TotalLoadedCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_classes", "total_unloaded_count"), + "JVM classes total unloaded", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Classes.TotalUnloadedCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm", "uptime_seconds"), + "JVM process uptime in seconds", + append(defaultNodeLabels, "type"), nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.JVM.Uptime) / 1000 + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return append(defaultNodeLabelValues(cluster, node), "mapped") + }, + }, + }, + gcCollectionMetrics: []*gcCollectionMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_gc", "collection_seconds_count"), + "Count of JVM GC runs", + append(defaultNodeLabels, "gc"), nil, + ), + Value: func(gcStats NodeStatsJVMGCCollectorResponse) float64 { + return float64(gcStats.CollectionCount) + }, + Labels: func(cluster string, node NodeStatsNodeResponse, collector string) []string { + return append(defaultNodeLabelValues(cluster, node), collector) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "jvm_gc", "collection_seconds_sum"), + "GC run time in seconds", + append(defaultNodeLabels, "gc"), nil, + ), + Value: func(gcStats NodeStatsJVMGCCollectorResponse) float64 { + return float64(gcStats.CollectionTime) / 1000 + }, + Labels: func(cluster string, node NodeStatsNodeResponse, collector string) []string { + return append(defaultNodeLabelValues(cluster, node), collector) + }, + }, + }, + osMetrics: []*nodeMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cpu_load_average_1m"), + "Shortterm load average", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return node.OS.CPU.LoadAvg.Load1 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cpu_load_average_5m"), + "Midterm load average", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return node.OS.CPU.LoadAvg.Load5 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cpu_load_average_15m"), + "Longterm load average", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return node.OS.CPU.LoadAvg.Load15 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cpu_percent"), + "Percent CPU used by OS", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.CPU.Percent) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "mem_free_in_bytes"), + "Amount of free physical memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Mem.Free) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "mem_used_in_bytes"), + "Amount of used physical memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Mem.Used) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "mem_actual_free_in_bytes"), + "Amount of free physical memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Mem.ActualFree) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "mem_actual_used_in_bytes"), + "Amount of used physical memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Mem.ActualUsed) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "mem_used_percent"), + "Percent of used physical memory", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return node.OS.Mem.UsedPercent + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "mem_total_in_bytes"), + "Amount of used physical memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Mem.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "mem_free_percent"), + "Percent of free physical memory", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return node.OS.Mem.FreePercent + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cgroup_cpu_cfs_period_micros"), + "CPU CFS period in microseconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.CGroup.CPU.CFSPeriod) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cgroup_cpu_cfs_quota_micros"), + "CPU CFS quota in microseconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.CGroup.CPU.CFSQuota) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cgroup_cpu_stat_number_of_elapsed_periods"), + "CPU CFS quota in microseconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.CGroup.CPU.Stat.NumberElapsed) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cgroup_cpu_stat_number_of_times_throttled"), + "CPU CFS quota in microseconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.CGroup.CPU.Stat.NumberThrottled) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cgroup_cpu_stat_time_throttled_nanos"), + "CPU CFS quota in microseconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.CGroup.CPU.Stat.TimeThrottled) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "cgroup_cpuacct_usage_nanos"), + "Cpuacct usage in nanos", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.CGroup.CPUAcct.Usage) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "swap_used_in_bytes"), + "Amount of used swap memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Swap.Used) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "swap_total_in_bytes"), + "Amount of total swap memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Swap.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "os", "swap_free_in_bytes"), + "Amount of free swap memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.OS.Swap.Free) + }, + Labels: defaultNodeLabelValues, + }, + }, + processMetrics: []*nodeMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "cpu_percent"), + "Percent CPU used by process", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.CPU.Percent) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "mem_resident_size_in_bytes"), + "Resident memory in use by process in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.Memory.Resident) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "mem_share_size_in_bytes"), + "Shared memory in use by process in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.Memory.Share) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "mem_total_virtual_in_bytes"), + "Total virtual memory used in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.Memory.TotalVirtual) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "timestamp"), + "Timestamp", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.Timestamp) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "open_files_descriptors"), + "Open file descriptors", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.OpenFD) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "max_files_descriptors"), + "Max file descriptors", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.MaxFD) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "process", "cpu_seconds_total"), + "Process CPU time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Process.CPU.Total) / 1000 + }, + Labels: func(cluster string, node NodeStatsNodeResponse) []string { + return defaultNodeLabelValues(cluster, node) + }, + }, + }, + breakerMetrics: []*breakerMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "breakers", "estimated_size_in_bytes"), + "Estimated size in bytes of breaker", + defaultBreakerLabels, nil, + ), + Value: func(breakerStats NodeStatsBreakersResponse) float64 { + return float64(breakerStats.EstimatedSize) + }, + Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { + return append(defaultNodeLabelValues(cluster, node), breaker) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "breakers", "limit_size_in_bytes"), + "Limit size in bytes for breaker", + defaultBreakerLabels, nil, + ), + Value: func(breakerStats NodeStatsBreakersResponse) float64 { + return float64(breakerStats.LimitSize) + }, + Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { + return append(defaultNodeLabelValues(cluster, node), breaker) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "breakers", "tripped"), + "tripped for breaker", + defaultBreakerLabels, nil, + ), + Value: func(breakerStats NodeStatsBreakersResponse) float64 { + return float64(breakerStats.Tripped) + }, + Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { + return append(defaultNodeLabelValues(cluster, node), breaker) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "breakers", "overhead"), + "Overhead of circuit breakers", + defaultBreakerLabels, nil, + ), + Value: func(breakerStats NodeStatsBreakersResponse) float64 { + return breakerStats.Overhead + }, + Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { + return append(defaultNodeLabelValues(cluster, node), breaker) + }, + }, + }, + indicesMetrics: []*nodeMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "fielddata_memory_size_in_bytes"), + "Field data cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FieldData.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "fielddata_evictions"), + "Evictions from field data", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FieldData.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "completion_size_in_bytes"), + "Completion in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Completion.Size) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "filter_cache_memory_size_in_bytes"), + "Filter cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FilterCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "filter_cache_evictions"), + "Evictions from filter cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FilterCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_memory_size_in_bytes"), + "Query cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_evictions"), + "Evictions from query cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_total_count"), + "Query cache total count", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.TotalCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_cache_size"), + "Query cache cache size", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.CacheSize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_cache_count"), + "Query cache cache count", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.CacheCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_hit_count"), + "Query cache count", + defaultCacheLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.HitCount) + }, + Labels: defaultCacheHitLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_miss_count"), + "Query miss count", + defaultCacheLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.MissCount) + }, + Labels: defaultCacheMissLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_cache_memory_size_in_bytes"), + "Request cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_cache_evictions"), + "Evictions from request cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_cache_count"), + "Request cache count", + defaultCacheLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.HitCount) + }, + Labels: defaultCacheHitLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_miss_count"), + "Request miss count", + defaultCacheLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.MissCount) + }, + Labels: defaultCacheMissLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "translog_operations"), + "Total translog operations", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Translog.Operations) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "translog_size_in_bytes"), + "Total translog size in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Translog.Size) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_time_seconds"), + "Total get time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.Time) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_total"), + "Total get", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_missing_time_seconds"), + "Total time of get missing in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.MissingTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_missing_total"), + "Total get missing", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.MissingTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_exists_time_seconds"), + "Total time get exists in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.ExistsTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_exists_total"), + "Total get exists operations", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.ExistsTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_refresh", "time_seconds_total"), + "Total time spent refreshing in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Refresh.TotalTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_refresh", "total"), + "Total refreshes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Refresh.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_query_time_seconds"), + "Total search query time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.QueryTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_query_total"), + "Total number of queries", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.QueryTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_fetch_time_seconds"), + "Total search fetch time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.FetchTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_fetch_total"), + "Total number of fetches", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.FetchTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_suggest_total"), + "Total number of suggests", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.SuggestTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_suggest_time_seconds"), + "Total suggest time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.SuggestTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_scroll_total"), + "Total number of scrolls", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.ScrollTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_scroll_time_seconds"), + "Total scroll time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.ScrollTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "docs_count"), + "Count of documents on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Docs.Count) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "docs_deleted"), + "Count of deleted documents on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Docs.Deleted) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_size_in_bytes"), + "Current size of stored index data in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Store.Size) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_throttle_time_seconds_total"), + "Throttle time for index store in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Store.ThrottleTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_memory_in_bytes"), + "Current memory size of segments in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.Memory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_count"), + "Count of index segments on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.Count) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_terms_memory_in_bytes"), + "Count of terms in memory for this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.TermsMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_index_writer_memory_in_bytes"), + "Count of memory for index writer on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.IndexWriterMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_max_unsafe_auto_id_timestamp"), + "Count of memory for index writer on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.MaxUnsafeAutoIDTimestamp) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_norms_memory_in_bytes"), + "Count of memory used by norms", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.NormsMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_stored_fields_memory_in_bytes"), + "Count of stored fields memory", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.StoredFieldsMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_doc_values_memory_in_bytes"), + "Count of doc values memory", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.DocValuesMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_fixed_bit_set_memory_in_bytes"), + "Count of fixed bit set", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.FixedBitSet) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_term_vectors_memory_in_bytes"), + "Term vectors memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.TermVectorsMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_points_memory_in_bytes"), + "Point values memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.PointsMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_version_map_memory_in_bytes"), + "Version map memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.VersionMapMemory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "flush_total"), + "Total flushes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Flush.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "flush_time_seconds"), + "Cumulative flush time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Flush.Time) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "warmer_total"), + "Total warmer count", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Warmer.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "warmer_time_seconds_total"), + "Total warmer time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Warmer.TotalTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "index_time_seconds_total"), + "Cumulative index time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.IndexTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "index_total"), + "Total index calls", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.IndexTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "delete_time_seconds_total"), + "Total time indexing delete in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.DeleteTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "delete_total"), + "Total indexing deletes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.DeleteTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "is_throttled"), + "Indexing throttling", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + if node.Indices.Indexing.IsThrottled { + return 1 + } + return 0 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "throttle_time_seconds_total"), + "Cumulative indexing throttling time", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.ThrottleTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total"), + "Total merges", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "current"), + "Current merges", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.Current) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "current_size_in_bytes"), + "Size of a current merges in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.CurrentSize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "docs_total"), + "Cumulative docs merged", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalDocs) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total_size_in_bytes"), + "Total merge size in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalSize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total_time_seconds_total"), + "Total time spent merging in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total_throttled_time_seconds"), + "Total throttled time of merges in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalThrottledTime) / 1000 + }, + Labels: defaultNodeLabelValues, + }, + }, + threadPoolMetrics: []*threadPoolMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "thread_pool", "completed_count"), + "Thread Pool operations completed", + defaultThreadPoolLabels, nil, + ), + Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { + return float64(threadPoolStats.Completed) + }, + Labels: defaultThreadPoolLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "thread_pool", "rejected_count"), + "Thread Pool operations rejected", + defaultThreadPoolLabels, nil, + ), + Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { + return float64(threadPoolStats.Rejected) + }, + Labels: defaultThreadPoolLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "thread_pool", "active_count"), + "Thread Pool threads active", + defaultThreadPoolLabels, nil, + ), + Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { + return float64(threadPoolStats.Active) + }, + Labels: defaultThreadPoolLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "thread_pool", "largest_count"), + "Thread Pool largest threads count", + defaultThreadPoolLabels, nil, + ), + Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { + return float64(threadPoolStats.Largest) + }, + Labels: defaultThreadPoolLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "thread_pool", "queue_count"), + "Thread Pool operations queued", + defaultThreadPoolLabels, nil, + ), + Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { + return float64(threadPoolStats.Queue) + }, + Labels: defaultThreadPoolLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "thread_pool", "threads_count"), + "Thread Pool current threads count", + defaultThreadPoolLabels, nil, + ), + Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { + return float64(threadPoolStats.Threads) + }, + Labels: defaultThreadPoolLabelValues, + }, + }, + filesystemDataMetrics: []*filesystemDataMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_data", "available_bytes"), + "Available space on block device in bytes", + defaultFilesystemDataLabels, nil, + ), + Value: func(fsStats NodeStatsFSDataResponse) float64 { + return float64(fsStats.Available) + }, + Labels: defaultFilesystemDataLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_data", "free_bytes"), + "Free space on block device in bytes", + defaultFilesystemDataLabels, nil, + ), + Value: func(fsStats NodeStatsFSDataResponse) float64 { + return float64(fsStats.Free) + }, + Labels: defaultFilesystemDataLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_data", "size_in_bytes"), + "Size of block device in bytes", + defaultFilesystemDataLabels, nil, + ), + Value: func(fsStats NodeStatsFSDataResponse) float64 { + return float64(fsStats.Total) + }, + Labels: defaultFilesystemDataLabelValues, + }, + }, + filesystemIODeviceMetrics: []*filesystemIODeviceMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "operations_count"), + "Count of disk operations", + defaultFilesystemIODeviceLabels, nil, + ), + Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { + return float64(fsIODeviceStats.Operations) + }, + Labels: defaultFilesystemIODeviceLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "read_operations_count"), + "Count of disk read operations", + defaultFilesystemIODeviceLabels, nil, + ), + Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { + return float64(fsIODeviceStats.ReadOperations) + }, + Labels: defaultFilesystemIODeviceLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "write_operations_count"), + "Count of disk write operations", + defaultFilesystemIODeviceLabels, nil, + ), + Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { + return float64(fsIODeviceStats.WriteOperations) + }, + Labels: defaultFilesystemIODeviceLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "read_size_kilobytes_sum"), + "Total kilobytes read from disk", + defaultFilesystemIODeviceLabels, nil, + ), + Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { + return float64(fsIODeviceStats.ReadSize) + }, + Labels: defaultFilesystemIODeviceLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "write_size_kilobytes_sum"), + "Total kilobytes written to disk", + defaultFilesystemIODeviceLabels, nil, + ), + Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { + return float64(fsIODeviceStats.WriteSize) + }, + Labels: defaultFilesystemIODeviceLabelValues, + }, + }, + } +} + +// Describe add metrics descriptions +func (c *Nodes) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range c.jvmMetrics { + ch <- metric.Desc + } + for _, metric := range c.gcCollectionMetrics { + ch <- metric.Desc + } + for _, metric := range c.breakerMetrics { + ch <- metric.Desc + } + for _, metric := range c.osMetrics { + ch <- metric.Desc + } + for _, metric := range c.processMetrics { + ch <- metric.Desc + } + for _, metric := range c.indicesMetrics { + ch <- metric.Desc + } + for _, metric := range c.transportMetrics { + ch <- metric.Desc + } + for _, metric := range c.threadPoolMetrics { + ch <- metric.Desc + } + for _, metric := range c.filesystemDataMetrics { + ch <- metric.Desc + } + for _, metric := range c.filesystemIODeviceMetrics { + ch <- metric.Desc + } + ch <- c.up.Desc() + ch <- c.totalScrapes.Desc() + ch <- c.jsonParseFailures.Desc() +} + +func (c *Nodes) fetchAndDecodeNodeStats() (nodeStatsResponse, error) { + var nsr nodeStatsResponse + + u := *c.url + + if c.all { + if c.local { + u.Path = path.Join(u.Path, "/_nodes/_local/stats") + } else { + u.Path = path.Join(u.Path, "/_nodes/stats") + } + } else { + if c.local { + u.Path = path.Join(u.Path, "/_nodes/_local", c.node, "stats") + } else { + u.Path = path.Join(u.Path, "/_nodes", c.node, "stats") + } + } + if len(c.nodeStats) != 0 { + u.Path = fmt.Sprintf("%s/%s", u.Path, strings.Join(c.nodeStats, ",")) + } + + res, err := c.client.Get(u.String()) + if err != nil { + return nsr, fmt.Errorf("failed to get cluster health from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return nsr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + c.jsonParseFailures.Inc() + return nsr, err + } + + if err := json.Unmarshal(bts, &nsr); err != nil { + c.jsonParseFailures.Inc() + return nsr, err + } + return nsr, nil +} + +func GetNodeID(client *http.Client, user, password, s string) (string, error) { + u, err := url.Parse(s) + if err != nil { + return "", fmt.Errorf("failed to parse URL %s: %s", s, err) + } + if user != "" && password != "" { + u.User = url.UserPassword(user, password) + } + var nsr nodeStatsResponse + u.Path = path.Join(u.Path, "/_nodes/_local/name") + res, err := client.Get(u.String()) + if err != nil { + return "", fmt.Errorf("failed to get node ID from %s: %s", u.String(), err) + } + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + if err := json.Unmarshal(bts, &nsr); err != nil { + return "", err + } + + // Only 1 should be returned + for id := range nsr.Nodes { + return id, nil + } + return "", nil +} + +func GetCatMaster(client *http.Client, user, password, s string) (string, error) { + u, err := url.Parse(s) + if err != nil { + return "", fmt.Errorf("failed to parse URL %s: %s", s, err) + } + if user != "" && password != "" { + u.User = url.UserPassword(user, password) + } + u.Path = path.Join(u.Path, "/_cat/master") + res, err := client.Get(u.String()) + if err != nil { + return "", fmt.Errorf("failed to get node ID from %s: %s", u.String(), err) + } + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + // NOTE: we are not going to read/discard r.Body under the assumption we'd prefer + // to let the underlying transport close the connection and re-establish a new one for + // future calls. + return "", fmt.Errorf("elasticsearch: Unable to retrieve master node information. API responded with status-code %d, expected %d", res.StatusCode, http.StatusOK) + } + response, err := io.ReadAll(res.Body) + + if err != nil { + return "", err + } + + masterID := strings.Split(string(response), " ")[0] + + return masterID, nil +} + +// Collect gets nodes metric values +func (c *Nodes) Collect(ch chan<- prometheus.Metric) { + c.totalScrapes.Inc() + defer func() { + ch <- c.up + ch <- c.totalScrapes + ch <- c.jsonParseFailures + }() + + nodeStatsResp, err := c.fetchAndDecodeNodeStats() + if err != nil { + c.up.Set(0) + log.Println("failed to fetch and decode node stats, err: ", err) + return + } + c.up.Set(1) + + for _, node := range nodeStatsResp.Nodes { + // Handle the node labels metric + roles := getRoles(node) + + for role, roleEnabled := range roles { + if roleEnabled { + metric := createRoleMetric(role) + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node), + metric.Labels(nodeStatsResp.ClusterName, node)..., + ) + } + } + + if isEnable("jvm", c.nodeStats) { + for _, metric := range c.jvmMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node), + metric.Labels(nodeStatsResp.ClusterName, node)..., + ) + } + + // GC Stats + for collector, gcStats := range node.JVM.GC.Collectors { + for _, metric := range c.gcCollectionMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(gcStats), + metric.Labels(nodeStatsResp.ClusterName, node, collector)..., + ) + } + } + } + + if isEnable("breaker", c.nodeStats) { + // Breaker stats + for breaker, bstats := range node.Breakers { + for _, metric := range c.breakerMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(bstats), + metric.Labels(nodeStatsResp.ClusterName, node, breaker)..., + ) + } + } + } + + if isEnable("process", c.nodeStats) { + // Process Stats + for _, metric := range c.processMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node), + metric.Labels(nodeStatsResp.ClusterName, node)..., + ) + } + } + + if isEnable("os", c.nodeStats) { + // OS Stats + for _, metric := range c.osMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node), + metric.Labels(nodeStatsResp.ClusterName, node)..., + ) + } + } + + if isEnable("fs", c.nodeStats) { + // File System Data Stats + for _, fsDataStats := range node.FS.Data { + for _, metric := range c.filesystemDataMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(fsDataStats), + metric.Labels(nodeStatsResp.ClusterName, node, fsDataStats.Mount, fsDataStats.Path)..., + ) + } + } + + // File System IO Device Stats + for _, fsIODeviceStats := range node.FS.IOStats.Devices { + for _, metric := range c.filesystemIODeviceMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(fsIODeviceStats), + metric.Labels(nodeStatsResp.ClusterName, node, fsIODeviceStats.DeviceName)..., + ) + } + } + } + + if isEnable("indices", c.nodeStats) { + // Indices Stats + for _, metric := range c.indicesMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node), + metric.Labels(nodeStatsResp.ClusterName, node)..., + ) + } + } + + if isEnable("thread_pool", c.nodeStats) { + // Thread Pool stats + for pool, pstats := range node.ThreadPool { + for _, metric := range c.threadPoolMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(pstats), + metric.Labels(nodeStatsResp.ClusterName, node, pool)..., + ) + } + } + } + + if isEnable("transport", c.nodeStats) { + // Transport Stats + for _, metric := range c.transportMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node), + metric.Labels(nodeStatsResp.ClusterName, node)..., + ) + } + } + } +} + +func isEnable(stat string, stats []string) bool { + for _, s := range stats { + if s == stat { + return true + } + } + return false +} diff --git a/inputs/elasticsearch/collector/nodes_response.go b/inputs/elasticsearch/collector/nodes_response.go new file mode 100644 index 00000000..b68d0d11 --- /dev/null +++ b/inputs/elasticsearch/collector/nodes_response.go @@ -0,0 +1,446 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" +) + +// nodeStatsResponse is a representation of an Elasticsearch Node Stats +type nodeStatsResponse struct { + ClusterName string `json:"cluster_name"` + Nodes map[string]NodeStatsNodeResponse +} + +// NodeStatsNodeResponse defines node stats information structure for nodes +type NodeStatsNodeResponse struct { + Name string `json:"name"` + Host string `json:"host"` + Timestamp int64 `json:"timestamp"` + TransportAddress string `json:"transport_address"` + Hostname string `json:"hostname"` + Roles []string `json:"roles"` + Attributes map[string]string `json:"attributes"` + Indices NodeStatsIndicesResponse `json:"indices"` + OS NodeStatsOSResponse `json:"os"` + Network NodeStatsNetworkResponse `json:"network"` + FS NodeStatsFSResponse `json:"fs"` + ThreadPool map[string]NodeStatsThreadPoolPoolResponse `json:"thread_pool"` + JVM NodeStatsJVMResponse `json:"jvm"` + Breakers map[string]NodeStatsBreakersResponse `json:"breakers"` + HTTP map[string]interface{} `json:"http"` + Transport NodeStatsTransportResponse `json:"transport"` + Process NodeStatsProcessResponse `json:"process"` +} + +// NodeStatsBreakersResponse is a representation of a statistics about the field data circuit breaker +type NodeStatsBreakersResponse struct { + EstimatedSize int64 `json:"estimated_size_in_bytes"` + LimitSize int64 `json:"limit_size_in_bytes"` + Overhead float64 `json:"overhead"` + Tripped int64 `json:"tripped"` +} + +// NodeStatsJVMResponse is a representation of a JVM stats, memory pool information, garbage collection, buffer pools, number of loaded/unloaded classes +type NodeStatsJVMResponse struct { + BufferPools map[string]NodeStatsJVMBufferPoolResponse `json:"buffer_pools"` + GC NodeStatsJVMGCResponse `json:"gc"` + Mem NodeStatsJVMMemResponse `json:"mem"` + Uptime int64 `json:"uptime_in_millis"` + Classes NodeStatsJVMClassesResponse `json:"classes"` + Threads NodeStatsJVMTreadsResponse `json:"threads"` + Timestamp int64 `json:"timestamp"` +} + +// NodeStatsJVMTreadsResponse defines node stats JVM threads information structure +type NodeStatsJVMTreadsResponse struct { + Count int64 `json:"count"` + PeakCount int64 `json:"peak_count"` +} + +// NodeStatsJVMClassesResponse defines node stats JVM classes information structure +type NodeStatsJVMClassesResponse struct { + CurrentLoadedCount int64 `json:"current_loaded_count"` + TotalLoadedCount int64 `json:"total_loaded_count"` + TotalUnloadedCount int64 `json:"total_unloaded_count"` +} + +// NodeStatsJVMGCResponse defines node stats JVM garbage collector information structure +type NodeStatsJVMGCResponse struct { + Collectors map[string]NodeStatsJVMGCCollectorResponse `json:"collectors"` +} + +// NodeStatsJVMGCCollectorResponse defines node stats JVM garbage collector collection information structure +type NodeStatsJVMGCCollectorResponse struct { + CollectionCount int64 `json:"collection_count"` + CollectionTime int64 `json:"collection_time_in_millis"` +} + +// NodeStatsJVMBufferPoolResponse defines node stats JVM buffer pool information structure +type NodeStatsJVMBufferPoolResponse struct { + Count int64 `json:"count"` + TotalCapacity int64 `json:"total_capacity_in_bytes"` + Used int64 `json:"used_in_bytes"` +} + +// NodeStatsJVMMemResponse defines node stats JVM memory information structure +type NodeStatsJVMMemResponse struct { + HeapCommitted int64 `json:"heap_committed_in_bytes"` + HeapUsed int64 `json:"heap_used_in_bytes"` + HeapMax int64 `json:"heap_max_in_bytes"` + HeapUsedPercent int64 `json:"heap_used_percent"` + NonHeapCommitted int64 `json:"non_heap_committed_in_bytes"` + NonHeapUsed int64 `json:"non_heap_used_in_bytes"` + Pools map[string]NodeStatsJVMMemPoolResponse `json:"pools"` +} + +// NodeStatsJVMMemPoolResponse defines node stats JVM memory pool information structure +type NodeStatsJVMMemPoolResponse struct { + Used int64 `json:"used_in_bytes"` + Max int64 `json:"max_in_bytes"` + PeakUsed int64 `json:"peak_used_in_bytes"` + PeakMax int64 `json:"peak_max_in_bytes"` +} + +// NodeStatsNetworkResponse defines node stats network information structure +type NodeStatsNetworkResponse struct { + TCP NodeStatsTCPResponse `json:"tcp"` +} + +// NodeStatsTransportResponse is a representation of a transport statistics about sent and received bytes in cluster communication +type NodeStatsTransportResponse struct { + ServerOpen int64 `json:"server_open"` + RxCount int64 `json:"rx_count"` + RxSize int64 `json:"rx_size_in_bytes"` + TxCount int64 `json:"tx_count"` + TxSize int64 `json:"tx_size_in_bytes"` +} + +// NodeStatsThreadPoolPoolResponse is a representation of a statistics about each thread pool, including current size, queue and rejected tasks +type NodeStatsThreadPoolPoolResponse struct { + Threads int64 `json:"threads"` + Queue int64 `json:"queue"` + Active int64 `json:"active"` + Rejected int64 `json:"rejected"` + Largest int64 `json:"largest"` + Completed int64 `json:"completed"` +} + +// NodeStatsTCPResponse defines node stats TCP information structure +type NodeStatsTCPResponse struct { + ActiveOpens int64 `json:"active_opens"` + PassiveOpens int64 `json:"passive_opens"` + CurrEstab int64 `json:"curr_estab"` + InSegs int64 `json:"in_segs"` + OutSegs int64 `json:"out_segs"` + RetransSegs int64 `json:"retrans_segs"` + EstabResets int64 `json:"estab_resets"` + AttemptFails int64 `json:"attempt_fails"` + InErrs int64 `json:"in_errs"` + OutRsts int64 `json:"out_rsts"` +} + +// NodeStatsIndicesResponse is a representation of a indices stats (size, document count, indexing and deletion times, search times, field cache size, merges and flushes) +type NodeStatsIndicesResponse struct { + Docs NodeStatsIndicesDocsResponse + Store NodeStatsIndicesStoreResponse + Indexing NodeStatsIndicesIndexingResponse + Merges NodeStatsIndicesMergesResponse + Get NodeStatsIndicesGetResponse + Search NodeStatsIndicesSearchResponse + FieldData NodeStatsIndicesCacheResponse `json:"fielddata"` + FilterCache NodeStatsIndicesCacheResponse `json:"filter_cache"` + QueryCache NodeStatsIndicesCacheResponse `json:"query_cache"` + RequestCache NodeStatsIndicesCacheResponse `json:"request_cache"` + Flush NodeStatsIndicesFlushResponse + Warmer NodeStatsIndicesWarmerResponse + Segments NodeStatsIndicesSegmentsResponse + Refresh NodeStatsIndicesRefreshResponse + Translog NodeStatsIndicesTranslogResponse + Completion NodeStatsIndicesCompletionResponse +} + +// NodeStatsIndicesDocsResponse defines node stats docs information structure for indices +type NodeStatsIndicesDocsResponse struct { + Count int64 `json:"count"` + Deleted int64 `json:"deleted"` +} + +// NodeStatsIndicesRefreshResponse defines node stats refresh information structure for indices +type NodeStatsIndicesRefreshResponse struct { + Total int64 `json:"total"` + TotalTime int64 `json:"total_time_in_millis"` +} + +// NodeStatsIndicesTranslogResponse defines node stats translog information structure for indices +type NodeStatsIndicesTranslogResponse struct { + Operations int64 `json:"operations"` + Size int64 `json:"size_in_bytes"` +} + +// NodeStatsIndicesCompletionResponse defines node stats completion information structure for indices +type NodeStatsIndicesCompletionResponse struct { + Size int64 `json:"size_in_bytes"` +} + +// NodeStatsIndicesSegmentsResponse defines node stats segments information structure for indices +type NodeStatsIndicesSegmentsResponse struct { + Count int64 `json:"count"` + Memory int64 `json:"memory_in_bytes"` + TermsMemory int64 `json:"terms_memory_in_bytes"` + IndexWriterMemory int64 `json:"index_writer_memory_in_bytes"` + NormsMemory int64 `json:"norms_memory_in_bytes"` + StoredFieldsMemory int64 `json:"stored_fields_memory_in_bytes"` + FixedBitSet int64 `json:"fixed_bit_set_memory_in_bytes"` + DocValuesMemory int64 `json:"doc_values_memory_in_bytes"` + TermVectorsMemory int64 `json:"term_vectors_memory_in_bytes"` + PointsMemory int64 `json:"points_memory_in_bytes"` + VersionMapMemory int64 `json:"version_map_memory_in_bytes"` + MaxUnsafeAutoIDTimestamp int64 `json:"max_unsafe_auto_id_timestamp"` +} + +// NodeStatsIndicesStoreResponse defines node stats store information structure for indices +type NodeStatsIndicesStoreResponse struct { + Size int64 `json:"size_in_bytes"` + ThrottleTime int64 `json:"throttle_time_in_millis"` +} + +// NodeStatsIndicesIndexingResponse defines node stats indexing information structure for indices +type NodeStatsIndicesIndexingResponse struct { + IndexTotal int64 `json:"index_total"` + IndexTime int64 `json:"index_time_in_millis"` + IndexCurrent int64 `json:"index_current"` + DeleteTotal int64 `json:"delete_total"` + DeleteTime int64 `json:"delete_time_in_millis"` + DeleteCurrent int64 `json:"delete_current"` + IsThrottled bool `json:"is_throttled"` + ThrottleTime int64 `json:"throttle_time_in_millis"` +} + +// NodeStatsIndicesMergesResponse defines node stats merges information structure for indices +type NodeStatsIndicesMergesResponse struct { + Current int64 `json:"current"` + CurrentDocs int64 `json:"current_docs"` + CurrentSize int64 `json:"current_size_in_bytes"` + Total int64 `json:"total"` + TotalDocs int64 `json:"total_docs"` + TotalSize int64 `json:"total_size_in_bytes"` + TotalTime int64 `json:"total_time_in_millis"` + TotalThrottledTime int64 `json:"total_throttled_time_in_millis"` +} + +// NodeStatsIndicesGetResponse defines node stats get information structure for indices +type NodeStatsIndicesGetResponse struct { + Total int64 `json:"total"` + Time int64 `json:"time_in_millis"` + ExistsTotal int64 `json:"exists_total"` + ExistsTime int64 `json:"exists_time_in_millis"` + MissingTotal int64 `json:"missing_total"` + MissingTime int64 `json:"missing_time_in_millis"` + Current int64 `json:"current"` +} + +// NodeStatsIndicesSearchResponse defines node stats search information structure for indices +type NodeStatsIndicesSearchResponse struct { + OpenContext int64 `json:"open_contexts"` + QueryTotal int64 `json:"query_total"` + QueryTime int64 `json:"query_time_in_millis"` + QueryCurrent int64 `json:"query_current"` + FetchTotal int64 `json:"fetch_total"` + FetchTime int64 `json:"fetch_time_in_millis"` + FetchCurrent int64 `json:"fetch_current"` + SuggestTotal int64 `json:"suggest_total"` + SuggestTime int64 `json:"suggest_time_in_millis"` + ScrollTotal int64 `json:"scroll_total"` + ScrollTime int64 `json:"scroll_time_in_millis"` +} + +// NodeStatsIndicesFlushResponse defines node stats flush information structure for indices +type NodeStatsIndicesFlushResponse struct { + Total int64 `json:"total"` + Time int64 `json:"total_time_in_millis"` +} + +// NodeStatsIndicesWarmerResponse defines node stats warmer information structure for indices +type NodeStatsIndicesWarmerResponse struct { + Total int64 `json:"total"` + TotalTime int64 `json:"total_time_in_millis"` +} + +// NodeStatsIndicesCacheResponse defines node stats cache information structure for indices +type NodeStatsIndicesCacheResponse struct { + Evictions int64 `json:"evictions"` + MemorySize int64 `json:"memory_size_in_bytes"` + CacheCount int64 `json:"cache_count"` + CacheSize int64 `json:"cache_size"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` + TotalCount int64 `json:"total_count"` +} + +// NodeStatsOSResponse is a representation of a operating system stats, load average, mem, swap +type NodeStatsOSResponse struct { + Timestamp int64 `json:"timestamp"` + Uptime int64 `json:"uptime_in_millis"` + // LoadAvg was an array of per-cpu values pre-2.0, and is a string in 2.0 + // Leaving this here in case we want to implement parsing logic later + LoadAvg json.RawMessage `json:"load_average"` + CPU NodeStatsOSCPUResponse `json:"cpu"` + Mem NodeStatsOSMemResponse `json:"mem"` + Swap NodeStatsOSSwapResponse `json:"swap"` + CGroup NodeStatsOSCGroupResponse `json:"cgroup"` +} + +// NodeStatsOSCGroupResponse defines node stats operating system cgroup structure +type NodeStatsOSCGroupResponse struct { + CPU NodeStatsOSCGroupCPUResponse `json:"cpu"` + CPUAcct NodeStatsOSCGroupCPUAcctResponse `json:"cpuacct"` + Memory NodeStatsOSCGroupMemoryResponse `json:"memory"` +} + +// NodeStatsOSCGroupCPUResponse defines node stats operating system cgroup CPU structure +type NodeStatsOSCGroupCPUResponse struct { + CFSPeriod int64 `json:"cfs_period_micros"` + CFSQuota int64 `json:"cfs_quota_micros"` + ControlGroup string `json:"control_group"` + Stat NodeStatsOSCGroupCPUStatResponse `json:"stat"` +} + +// NodeStatsOSCGroupCPUStatResponse defines node stats operating system cgroup CPU stat structure +type NodeStatsOSCGroupCPUStatResponse struct { + NumberThrottled int64 `json:"number_of_times_throttled"` + TimeThrottled int64 `json:"time_throttled_nanos"` + NumberElapsed int64 `json:"number_of_elapsed_periods"` +} + +// NodeStatsOSCGroupCPUAcctResponse defines node stats operating system cgroup CPU accounting structure +type NodeStatsOSCGroupCPUAcctResponse struct { + Usage int64 `json:"usage_nanos"` + ControlGroup string `json:"control_group"` +} + +// NodeStatsOSCGroupMemoryResponse defines node stats operating system cgroup memory structure +type NodeStatsOSCGroupMemoryResponse struct { + Limit string `json:"limit_in_bytes"` + Usage string `json:"usage_in_bytes"` + ControlGroup string `json:"control_group"` +} + +// NodeStatsOSMemResponse defines node stats operating system memory usage structure +type NodeStatsOSMemResponse struct { + Free int64 `json:"free_in_bytes"` + Used int64 `json:"used_in_bytes"` + ActualFree int64 `json:"actual_free_in_bytes"` + ActualUsed int64 `json:"actual_used_in_bytes"` + UsedPercent float64 `json:"used_percent"` + Total int64 `json:"total_in_bytes"` + FreePercent float64 `json:"free_percent"` +} + +// NodeStatsOSSwapResponse defines node stats operating system swap usage structure +type NodeStatsOSSwapResponse struct { + Used int64 `json:"used_in_bytes"` + Free int64 `json:"free_in_bytes"` + Total int64 `json:"total_in_bytes"` +} + +// NodeStatsOSCPUResponse defines node stats operating system CPU usage structure +type NodeStatsOSCPUResponse struct { + LoadAvg NodeStatsOSCPULoadResponse `json:"load_average"` + Percent int64 `json:"percent"` +} + +// NodeStatsOSCPULoadResponse defines node stats operating system CPU load structure +type NodeStatsOSCPULoadResponse struct { + Load1 float64 `json:"1m"` + Load5 float64 `json:"5m"` + Load15 float64 `json:"15m"` +} + +// NodeStatsProcessResponse is a representation of a process statistics, memory consumption, cpu usage, open file descriptors +type NodeStatsProcessResponse struct { + Timestamp int64 `json:"timestamp"` + OpenFD int64 `json:"open_file_descriptors"` + MaxFD int64 `json:"max_file_descriptors"` + CPU NodeStatsProcessCPUResponse `json:"cpu"` + Memory NodeStatsProcessMemResponse `json:"mem"` +} + +// NodeStatsProcessMemResponse defines node stats process memory usage structure +type NodeStatsProcessMemResponse struct { + Resident int64 `json:"resident_in_bytes"` + Share int64 `json:"share_in_bytes"` + TotalVirtual int64 `json:"total_virtual_in_bytes"` +} + +// NodeStatsProcessCPUResponse defines node stats process CPU usage structure +type NodeStatsProcessCPUResponse struct { + Percent int64 `json:"percent"` + Total int64 `json:"total_in_millis"` +} + +// NodeStatsHTTPResponse defines node stats HTTP connections structure +type NodeStatsHTTPResponse struct { + CurrentOpen int64 `json:"current_open"` + TotalOpen int64 `json:"total_open"` +} + +// NodeStatsFSResponse is a representation of a file system information, data path, free disk space, read/write stats +type NodeStatsFSResponse struct { + Timestamp int64 `json:"timestamp"` + Data []NodeStatsFSDataResponse `json:"data"` + IOStats NodeStatsFSIOStatsResponse `json:"io_stats"` +} + +// NodeStatsFSDataResponse defines node stats filesystem data structure +type NodeStatsFSDataResponse struct { + Path string `json:"path"` + Mount string `json:"mount"` + Device string `json:"dev"` + Total int64 `json:"total_in_bytes"` + Free int64 `json:"free_in_bytes"` + Available int64 `json:"available_in_bytes"` +} + +// NodeStatsFSIOStatsResponse defines node stats filesystem device structure +type NodeStatsFSIOStatsResponse struct { + Devices []NodeStatsFSIOStatsDeviceResponse `json:"devices"` +} + +// NodeStatsFSIOStatsDeviceResponse is a representation of a node stat filesystem device +type NodeStatsFSIOStatsDeviceResponse struct { + DeviceName string `json:"device_name"` + Operations int64 `json:"operations"` + ReadOperations int64 `json:"read_operations"` + WriteOperations int64 `json:"write_operations"` + ReadSize int64 `json:"read_kilobytes"` + WriteSize int64 `json:"write_kilobytes"` +} + +// ClusterHealthResponse is a representation of a Elasticsearch Cluster Health +type ClusterHealthResponse struct { + ActivePrimaryShards int64 `json:"active_primary_shards"` + ActiveShards int64 `json:"active_shards"` + ClusterName string `json:"cluster_name"` + DelayedUnassignedShards int64 `json:"delayed_unassigned_shards"` + InitializingShards int64 `json:"initializing_shards"` + NumberOfDataNodes int64 `json:"number_of_data_nodes"` + NumberOfInFlightFetch int64 `json:"number_of_in_flight_fetch"` + NumberOfNodes int64 `json:"number_of_nodes"` + NumberOfPendingTasks int64 `json:"number_of_pending_tasks"` + RelocatingShards int64 `json:"relocating_shards"` + Status string `json:"status"` + TimedOut bool `json:"timed_out"` + UnassignedShards int64 `json:"unassigned_shards"` +} diff --git a/inputs/elasticsearch/collector/nodes_test.go b/inputs/elasticsearch/collector/nodes_test.go new file mode 100644 index 00000000..d2fc9c77 --- /dev/null +++ b/inputs/elasticsearch/collector/nodes_test.go @@ -0,0 +1,171 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" +) + +func TestNodesStats(t *testing.T) { + for _, ver := range testElasticsearchVersions { + filename := fmt.Sprintf("../fixtures/nodestats/%s.json", ver) + data, _ := os.ReadFile(filename) + + handlers := map[string]http.Handler{ + "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, err := w.Write(data); err != nil { + t.Fatalf("failed write: %s", err) + } + }), + "basicauth": &basicAuth{ + User: "elastic", + Pass: "changeme", + Next: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, err := w.Write(data); err != nil { + t.Fatalf("failed write: %s", err) + } + }), + }, + } + + for hn, handler := range handlers { + t.Run(fmt.Sprintf("%s/%s", hn, ver), func(t *testing.T) { + + ts := httptest.NewServer(handler) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + u.User = url.UserPassword("elastic", "changeme") + nodeStats := make([]string, 0) + c := NewNodes(http.DefaultClient, u, true, "_local", false, nodeStats) + nsr, err := c.fetchAndDecodeNodeStats() + if err != nil { + t.Fatalf("Failed to fetch or decode node stats: %s", err) + } + t.Logf("[%s/%s] Node Stats Response: %+v", hn, ver, nsr) + // TODO(@sysadmind): Add multinode fixture + if nsr.ClusterName == "multinode" { + for _, node := range nsr.Nodes { + labels := defaultNodeLabelValues(nsr.ClusterName, node) + esMasterNode := labels[3] + esDataNode := labels[4] + esIngestNode := labels[5] + esClientNode := labels[6] + t.Logf( + "Node: %s - Master: %s - Data: %s - Ingest: %s - Client: %s", + node.Name, + esMasterNode, + esDataNode, + esIngestNode, + esClientNode, + ) + if strings.HasPrefix(node.Name, "elasticmaster") { + if esMasterNode != "true" { + t.Errorf("Master should be master") + } + if esDataNode == "true" { + t.Errorf("Master should be not data") + } + if esIngestNode == "true" { + t.Errorf("Master should be not ingest") + } + } + if strings.HasPrefix(node.Name, "elasticdata") { + if esMasterNode == "true" { + t.Errorf("Data should not be master") + } + if esDataNode != "true" { + t.Errorf("Data should be data") + } + if esIngestNode == "true" { + t.Errorf("Data should be not ingest") + } + } + if strings.HasPrefix(node.Name, "elasticin") { + if esMasterNode == "true" { + t.Errorf("Ingest should not be master") + } + if esDataNode == "true" { + t.Errorf("Ingest should be data") + } + if esIngestNode != "true" { + t.Errorf("Ingest should be not ingest") + } + } + if strings.HasPrefix(node.Name, "elasticcli") { + if esMasterNode == "true" { + t.Errorf("CLI should not be master") + } + if esDataNode == "true" { + t.Errorf("CLI should be data") + } + if esIngestNode == "true" { + t.Errorf("CLI should be not ingest") + } + } + } + } + }) + } + } +} + +type basicAuth struct { + User string + Pass string + Next http.Handler +} + +func (h *basicAuth) checkAuth(_ http.ResponseWriter, r *http.Request) bool { + s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) + if len(s) != 2 { + return false + } + + b, err := base64.StdEncoding.DecodeString(s[1]) + if err != nil { + return false + } + + pair := strings.SplitN(string(b), ":", 2) + if len(pair) != 2 { + return false + } + + if h.User == pair[0] && h.Pass == pair[1] { + return true + } + return false +} + +func (h *basicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !h.checkAuth(w, r) { + w.Header().Set("WWW-Authenticate", "Basic realm=\"ES\"") + w.WriteHeader(401) + w.Write([]byte("401 Unauthorized\n")) + return + } + + h.Next.ServeHTTP(w, r) +} diff --git a/inputs/elasticsearch/collector/shards.go b/inputs/elasticsearch/collector/shards.go new file mode 100644 index 00000000..236ef5bb --- /dev/null +++ b/inputs/elasticsearch/collector/shards.go @@ -0,0 +1,205 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "path" + + "flashcat.cloud/categraf/inputs/elasticsearch/pkg/clusterinfo" + "github.com/prometheus/client_golang/prometheus" +) + +// ShardResponse has shard's node and index info +type ShardResponse struct { + Index string `json:"index"` + Shard string `json:"shard"` + State string `json:"state"` + Node string `json:"node"` +} + +// Shards information struct +type Shards struct { + client *http.Client + url *url.URL + clusterInfoCh chan *clusterinfo.Response + lastClusterInfo *clusterinfo.Response + + nodeShardMetrics []*nodeShardMetric + jsonParseFailures prometheus.Counter +} + +// ClusterLabelUpdates returns a pointer to a channel to receive cluster info updates. It implements the +// (not exported) clusterinfo.consumer interface +func (s *Shards) ClusterLabelUpdates() *chan *clusterinfo.Response { + return &s.clusterInfoCh +} + +// String implements the stringer interface. It is part of the clusterinfo.consumer interface +func (s *Shards) String() string { + return namespace + "shards" +} + +type nodeShardMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(shards float64) float64 + Labels labels +} + +// NewShards defines Shards Prometheus metrics +func NewShards(client *http.Client, url *url.URL) *Shards { + + nodeLabels := labels{ + keys: func(...string) []string { + return []string{"node", "cluster"} + }, + values: func(lastClusterinfo *clusterinfo.Response, s ...string) []string { + if lastClusterinfo != nil { + return append(s, lastClusterinfo.ClusterName) + } + // this shouldn't happen, as the clusterinfo Retriever has a blocking + // Run method. It blocks until the first clusterinfo call has succeeded + return append(s, "unknown_cluster") + }, + } + + shards := &Shards{ + client: client, + url: url, + + clusterInfoCh: make(chan *clusterinfo.Response), + lastClusterInfo: &clusterinfo.Response{ + ClusterName: "unknown_cluster", + }, + + nodeShardMetrics: []*nodeShardMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "node_shards", "total"), + "Total shards per node", + nodeLabels.keys(), nil, + ), + Value: func(shards float64) float64 { + return shards + }, + Labels: nodeLabels, + }}, + + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "node_shards", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + } + + // start go routine to fetch clusterinfo updates and save them to lastClusterinfo + go func() { + log.Println("starting cluster info receive loop") + for ci := range shards.clusterInfoCh { + if ci != nil { + log.Println("received cluster info update, cluster ", ci.ClusterName) + shards.lastClusterInfo = ci + } + } + log.Println("exiting cluster info receive loop") + }() + + return shards +} + +// Describe Shards +func (s *Shards) Describe(ch chan<- *prometheus.Desc) { + ch <- s.jsonParseFailures.Desc() + + for _, metric := range s.nodeShardMetrics { + ch <- metric.Desc + } +} + +func (s *Shards) getAndParseURL(u *url.URL) ([]ShardResponse, error) { + res, err := s.client.Get(u.String()) + if err != nil { + return nil, fmt.Errorf("failed to get from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + var sfr []ShardResponse + if err := json.NewDecoder(res.Body).Decode(&sfr); err != nil { + s.jsonParseFailures.Inc() + return nil, err + } + return sfr, nil +} + +func (s *Shards) fetchAndDecodeShards() ([]ShardResponse, error) { + + u := *s.url + u.Path = path.Join(u.Path, "/_cat/shards") + q := u.Query() + q.Set("format", "json") + u.RawQuery = q.Encode() + sfr, err := s.getAndParseURL(&u) + if err != nil { + return sfr, err + } + return sfr, err +} + +// Collect number of shards on each node +func (s *Shards) Collect(ch chan<- prometheus.Metric) { + + defer func() { + ch <- s.jsonParseFailures + }() + + sr, err := s.fetchAndDecodeShards() + if err != nil { + log.Println("failed to fetch and decode node shards stats, err: ", err) + return + } + + nodeShards := make(map[string]float64) + + for _, shard := range sr { + if shard.State == "STARTED" { + nodeShards[shard.Node]++ + } + } + + for node, shards := range nodeShards { + for _, metric := range s.nodeShardMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(shards), + metric.Labels.values(s.lastClusterInfo, node)..., + ) + } + } +} diff --git a/inputs/elasticsearch/collector/slm.go b/inputs/elasticsearch/collector/slm.go new file mode 100644 index 00000000..a6fac11c --- /dev/null +++ b/inputs/elasticsearch/collector/slm.go @@ -0,0 +1,386 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +type policyMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(policyStats PolicyStats) float64 + Labels func(policyStats PolicyStats) []string +} + +type slmMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(slmStats SLMStatsResponse) float64 +} + +type slmStatusMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(slmStatus SLMStatusResponse, operationMode string) float64 + Labels func(operationMode string) []string +} + +var ( + defaultPolicyLabels = []string{"policy"} + defaultPolicyLabelValues = func(policyStats PolicyStats) []string { + return []string{policyStats.Policy} + } + + statuses = []string{"RUNNING", "STOPPING", "STOPPED"} +) + +// SLM information struct +type SLM struct { + client *http.Client + url *url.URL + + up prometheus.Gauge + totalScrapes, jsonParseFailures prometheus.Counter + + slmMetrics []*slmMetric + policyMetrics []*policyMetric + slmStatusMetric *slmStatusMetric +} + +// NewSLM defines SLM Prometheus metrics +func NewSLM(client *http.Client, url *url.URL) *SLM { + return &SLM{ + client: client, + url: url, + + up: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "slm_stats", "up"), + Help: "Was the last scrape of the Elasticsearch SLM endpoint successful.", + }), + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "slm_stats", "total_scrapes"), + Help: "Current total Elasticsearch SLM scrapes.", + }), + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "slm_stats", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + slmMetrics: []*slmMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "retention_runs_total"), + "Total retention runs", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.RetentionRuns) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "retention_failed_total"), + "Total failed retention runs", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.RetentionFailed) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "retention_timed_out_total"), + "Total timed out retention runs", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.RetentionTimedOut) + }, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "retention_deletion_time_seconds"), + "Retention run deletion time", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.RetentionDeletionTimeMillis) / 1000 + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "total_snapshots_taken_total"), + "Total snapshots taken", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.TotalSnapshotsTaken) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "total_snapshots_failed_total"), + "Total snapshots failed", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.TotalSnapshotsFailed) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "total_snapshots_deleted_total"), + "Total snapshots deleted", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.TotalSnapshotsDeleted) + }, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "total_snapshot_deletion_failures_total"), + "Total snapshot deletion failures", + nil, nil, + ), + Value: func(slmStats SLMStatsResponse) float64 { + return float64(slmStats.TotalSnapshotDeletionFailures) + }, + }, + }, + policyMetrics: []*policyMetric{ + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "snapshots_taken_total"), + "Total snapshots taken", + defaultPolicyLabels, nil, + ), + Value: func(policyStats PolicyStats) float64 { + return float64(policyStats.SnapshotsTaken) + }, + Labels: defaultPolicyLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "snapshots_failed_total"), + "Total snapshots failed", + defaultPolicyLabels, nil, + ), + Value: func(policyStats PolicyStats) float64 { + return float64(policyStats.SnapshotsFailed) + }, + Labels: defaultPolicyLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "snapshots_deleted_total"), + "Total snapshots deleted", + defaultPolicyLabels, nil, + ), + Value: func(policyStats PolicyStats) float64 { + return float64(policyStats.SnapshotsDeleted) + }, + Labels: defaultPolicyLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "snapshot_deletion_failures_total"), + "Total snapshot deletion failures", + defaultPolicyLabels, nil, + ), + Value: func(policyStats PolicyStats) float64 { + return float64(policyStats.SnapshotDeletionFailures) + }, + Labels: defaultPolicyLabelValues, + }, + }, + slmStatusMetric: &slmStatusMetric{ + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "slm_stats", "operation_mode"), + "Operating status of SLM", + []string{"operation_mode"}, nil, + ), + Value: func(slmStatus SLMStatusResponse, operationMode string) float64 { + if slmStatus.OperationMode == operationMode { + return 1 + } + return 0 + }, + }, + } +} + +// Describe adds SLM metrics descriptions +func (s *SLM) Describe(ch chan<- *prometheus.Desc) { + ch <- s.slmStatusMetric.Desc + + for _, metric := range s.slmMetrics { + ch <- metric.Desc + } + + for _, metric := range s.policyMetrics { + ch <- metric.Desc + } + + ch <- s.up.Desc() + ch <- s.totalScrapes.Desc() + ch <- s.jsonParseFailures.Desc() +} + +func (s *SLM) fetchAndDecodeSLMStats() (SLMStatsResponse, error) { + var ssr SLMStatsResponse + + u := *s.url + u.Path = path.Join(u.Path, "/_slm/stats") + res, err := s.client.Get(u.String()) + if err != nil { + return ssr, fmt.Errorf("failed to get slm stats health from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return ssr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + s.jsonParseFailures.Inc() + return ssr, err + } + + if err := json.Unmarshal(bts, &ssr); err != nil { + s.jsonParseFailures.Inc() + return ssr, err + } + + return ssr, nil +} + +func (s *SLM) fetchAndDecodeSLMStatus() (SLMStatusResponse, error) { + var ssr SLMStatusResponse + + u := *s.url + u.Path = path.Join(u.Path, "/_slm/status") + res, err := s.client.Get(u.String()) + if err != nil { + return ssr, fmt.Errorf("failed to get slm status from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return ssr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + s.jsonParseFailures.Inc() + return ssr, err + } + + if err := json.Unmarshal(bts, &ssr); err != nil { + s.jsonParseFailures.Inc() + return ssr, err + } + + return ssr, nil +} + +// Collect gets SLM metric values +func (s *SLM) Collect(ch chan<- prometheus.Metric) { + s.totalScrapes.Inc() + defer func() { + ch <- s.up + ch <- s.totalScrapes + ch <- s.jsonParseFailures + }() + + slmStatusResp, err := s.fetchAndDecodeSLMStatus() + if err != nil { + s.up.Set(0) + log.Println("failed to fetch and decode slm status, err: ", err) + return + } + + slmStatsResp, err := s.fetchAndDecodeSLMStats() + if err != nil { + s.up.Set(0) + log.Println("failed to fetch and decode slm stats, err: ", err) + return + } + + s.up.Set(1) + + for _, status := range statuses { + ch <- prometheus.MustNewConstMetric( + s.slmStatusMetric.Desc, + s.slmStatusMetric.Type, + s.slmStatusMetric.Value(slmStatusResp, status), + status, + ) + } + + for _, metric := range s.slmMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(slmStatsResp), + ) + } + + for _, metric := range s.policyMetrics { + for _, policy := range slmStatsResp.PolicyStats { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(policy), + metric.Labels(policy)..., + ) + } + } +} diff --git a/inputs/elasticsearch/collector/slm_response.go b/inputs/elasticsearch/collector/slm_response.go new file mode 100644 index 00000000..b1cfc1b1 --- /dev/null +++ b/inputs/elasticsearch/collector/slm_response.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +// SLMStatsResponse is a representation of the SLM stats +type SLMStatsResponse struct { + RetentionRuns int64 `json:"retention_runs"` + RetentionFailed int64 `json:"retention_failed"` + RetentionTimedOut int64 `json:"retention_timed_out"` + RetentionDeletionTime string `json:"retention_deletion_time"` + RetentionDeletionTimeMillis int64 `json:"retention_deletion_time_millis"` + TotalSnapshotsTaken int64 `json:"total_snapshots_taken"` + TotalSnapshotsFailed int64 `json:"total_snapshots_failed"` + TotalSnapshotsDeleted int64 `json:"total_snapshots_deleted"` + TotalSnapshotDeletionFailures int64 `json:"total_snapshot_deletion_failures"` + PolicyStats []PolicyStats `json:"policy_stats"` +} + +// PolicyStats is a representation of SLM stats for specific policies +type PolicyStats struct { + Policy string `json:"policy"` + SnapshotsTaken int64 `json:"snapshots_taken"` + SnapshotsFailed int64 `json:"snapshots_failed"` + SnapshotsDeleted int64 `json:"snapshots_deleted"` + SnapshotDeletionFailures int64 `json:"snapshot_deletion_failures"` +} + +// SLMStatusResponse is a representation of the SLM status +type SLMStatusResponse struct { + OperationMode string `json:"operation_mode"` +} diff --git a/inputs/elasticsearch/collector/slm_test.go b/inputs/elasticsearch/collector/slm_test.go new file mode 100644 index 00000000..957b1050 --- /dev/null +++ b/inputs/elasticsearch/collector/slm_test.go @@ -0,0 +1,63 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestSLM(t *testing.T) { + // Testcases created using: + + // docker run -d -p 9200:9200 -e discovery.type=single-node -e path.repo=/tmp/backups docker.elastic.co/elasticsearch/elasticsearch:7.15.0-arm64 + // curl -XPUT http://127.0.0.1:9200/_snapshot/my_repository -H 'Content-Type: application/json' -d '{"type":"url","settings":{"url":"file:/tmp/backups"}}' + // curl -XPUT http://127.0.0.1:9200/_slm/policy/everything -H 'Content-Type: application/json' -d '{"schedule":"0 */15 * * * ?","name":"","repository":"my_repository","config":{"indices":".*","include_global_state":true,"ignore_unavailable":true},"retention":{"expire_after":"7d"}}' + // curl http://127.0.0.1:9200/_slm/stats (Numbers manually tweaked) + + tcs := map[string]string{ + "7.15.0": `{"retention_runs":9,"retention_failed":0,"retention_timed_out":0,"retention_deletion_time":"1.2m","retention_deletion_time_millis":72491,"total_snapshots_taken":103,"total_snapshots_failed":2,"total_snapshots_deleted":20,"total_snapshot_deletion_failures":0,"policy_stats":[{"policy":"everything","snapshots_taken":50,"snapshots_failed":2,"snapshots_deleted":20,"snapshot_deletion_failures":0}]}`, + } + for ver, out := range tcs { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + s := NewSLM(http.DefaultClient, u) + stats, err := s.fetchAndDecodeSLMStats() + if err != nil { + t.Fatalf("Failed to fetch or decode snapshots stats: %s", err) + } + t.Logf("[%s] SLM Response: %+v", ver, stats) + slmStats := stats + policyStats := stats.PolicyStats[0] + + if slmStats.TotalSnapshotsTaken != 103 { + t.Errorf("Bad number of total snapshots taken") + } + + if policyStats.SnapshotsTaken != 50 { + t.Errorf("Bad number of policy snapshots taken") + } + } + +} diff --git a/inputs/elasticsearch/collector/snapshots.go b/inputs/elasticsearch/collector/snapshots.go new file mode 100644 index 00000000..94925a2a --- /dev/null +++ b/inputs/elasticsearch/collector/snapshots.go @@ -0,0 +1,302 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + + "github.com/prometheus/client_golang/prometheus" +) + +type snapshotMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(snapshotStats SnapshotStatDataResponse) float64 + Labels func(repositoryName string, snapshotStats SnapshotStatDataResponse) []string +} + +type repositoryMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(snapshotsStats SnapshotStatsResponse) float64 + Labels func(repositoryName string) []string +} + +var ( + defaultSnapshotLabels = []string{"repository", "state", "version"} + defaultSnapshotLabelValues = func(repositoryName string, snapshotStats SnapshotStatDataResponse) []string { + return []string{repositoryName, snapshotStats.State, snapshotStats.Version} + } + defaultSnapshotRepositoryLabels = []string{"repository"} + defaultSnapshotRepositoryLabelValues = func(repositoryName string) []string { + return []string{repositoryName} + } +) + +// Snapshots information struct +type Snapshots struct { + client *http.Client + url *url.URL + + snapshotMetrics []*snapshotMetric + repositoryMetrics []*repositoryMetric +} + +// NewSnapshots defines Snapshots Prometheus metrics +func NewSnapshots(client *http.Client, url *url.URL) *Snapshots { + return &Snapshots{ + client: client, + url: url, + + snapshotMetrics: []*snapshotMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_number_of_indices"), + "Number of indices in the last snapshot", + defaultSnapshotLabels, nil, + ), + Value: func(snapshotStats SnapshotStatDataResponse) float64 { + return float64(len(snapshotStats.Indices)) + }, + Labels: defaultSnapshotLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_start_time_timestamp"), + "Last snapshot start timestamp", + defaultSnapshotLabels, nil, + ), + Value: func(snapshotStats SnapshotStatDataResponse) float64 { + return float64(snapshotStats.StartTimeInMillis / 1000) + }, + Labels: defaultSnapshotLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_end_time_timestamp"), + "Last snapshot end timestamp", + defaultSnapshotLabels, nil, + ), + Value: func(snapshotStats SnapshotStatDataResponse) float64 { + return float64(snapshotStats.EndTimeInMillis / 1000) + }, + Labels: defaultSnapshotLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_number_of_failures"), + "Last snapshot number of failures", + defaultSnapshotLabels, nil, + ), + Value: func(snapshotStats SnapshotStatDataResponse) float64 { + return float64(len(snapshotStats.Failures)) + }, + Labels: defaultSnapshotLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_total_shards"), + "Last snapshot total shards", + defaultSnapshotLabels, nil, + ), + Value: func(snapshotStats SnapshotStatDataResponse) float64 { + return float64(snapshotStats.Shards.Total) + }, + Labels: defaultSnapshotLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_failed_shards"), + "Last snapshot failed shards", + defaultSnapshotLabels, nil, + ), + Value: func(snapshotStats SnapshotStatDataResponse) float64 { + return float64(snapshotStats.Shards.Failed) + }, + Labels: defaultSnapshotLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_successful_shards"), + "Last snapshot successful shards", + defaultSnapshotLabels, nil, + ), + Value: func(snapshotStats SnapshotStatDataResponse) float64 { + return float64(snapshotStats.Shards.Successful) + }, + Labels: defaultSnapshotLabelValues, + }, + }, + repositoryMetrics: []*repositoryMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "number_of_snapshots"), + "Number of snapshots in a repository", + defaultSnapshotRepositoryLabels, nil, + ), + Value: func(snapshotsStats SnapshotStatsResponse) float64 { + return float64(len(snapshotsStats.Snapshots)) + }, + Labels: defaultSnapshotRepositoryLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "oldest_snapshot_timestamp"), + "Timestamp of the oldest snapshot", + defaultSnapshotRepositoryLabels, nil, + ), + Value: func(snapshotsStats SnapshotStatsResponse) float64 { + if len(snapshotsStats.Snapshots) == 0 { + return 0 + } + return float64(snapshotsStats.Snapshots[0].StartTimeInMillis / 1000) + }, + Labels: defaultSnapshotRepositoryLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "snapshot_stats", "latest_snapshot_timestamp_seconds"), + "Timestamp of the latest SUCCESS or PARTIAL snapshot", + defaultSnapshotRepositoryLabels, nil, + ), + Value: func(snapshotsStats SnapshotStatsResponse) float64 { + for i := len(snapshotsStats.Snapshots) - 1; i >= 0; i-- { + var snap = snapshotsStats.Snapshots[i] + if snap.State == "SUCCESS" || snap.State == "PARTIAL" { + return float64(snap.StartTimeInMillis / 1000) + } + } + return 0 + }, + Labels: defaultSnapshotRepositoryLabelValues, + }, + }, + } +} + +// Describe add Snapshots metrics descriptions +func (s *Snapshots) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range s.snapshotMetrics { + ch <- metric.Desc + } + for _, metric := range s.repositoryMetrics { + ch <- metric.Desc + } + +} + +func (s *Snapshots) getAndParseURL(u *url.URL, data interface{}) error { + res, err := s.client.Get(u.String()) + if err != nil { + return fmt.Errorf("failed to get from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return err + } + + if err := json.Unmarshal(bts, data); err != nil { + return err + } + return nil +} + +func (s *Snapshots) fetchAndDecodeSnapshotsStats() (map[string]SnapshotStatsResponse, error) { + mssr := make(map[string]SnapshotStatsResponse) + + u := *s.url + u.Path = path.Join(u.Path, "/_snapshot") + var srr SnapshotRepositoriesResponse + err := s.getAndParseURL(&u, &srr) + if err != nil { + return nil, err + } + for repository := range srr { + u := *s.url + u.Path = path.Join(u.Path, "/_snapshot", repository, "/_all") + var ssr SnapshotStatsResponse + err := s.getAndParseURL(&u, &ssr) + if err != nil { + continue + } + mssr[repository] = ssr + } + + return mssr, nil +} + +// Collect gets Snapshots metric values +func (s *Snapshots) Collect(ch chan<- prometheus.Metric) { + + // indices + snapshotsStatsResp, err := s.fetchAndDecodeSnapshotsStats() + if err != nil { + log.Println("failed to fetch and decode snapshot stats, err: ", err) + return + } + + // Snapshots stats + for repositoryName, snapshotStats := range snapshotsStatsResp { + for _, metric := range s.repositoryMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(snapshotStats), + metric.Labels(repositoryName)..., + ) + } + if len(snapshotStats.Snapshots) == 0 { + continue + } + + lastSnapshot := snapshotStats.Snapshots[len(snapshotStats.Snapshots)-1] + for _, metric := range s.snapshotMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(lastSnapshot), + metric.Labels(repositoryName, lastSnapshot)..., + ) + } + } +} diff --git a/inputs/elasticsearch/collector/snapshots_reponse.go b/inputs/elasticsearch/collector/snapshots_reponse.go new file mode 100644 index 00000000..09bc57f8 --- /dev/null +++ b/inputs/elasticsearch/collector/snapshots_reponse.go @@ -0,0 +1,47 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import "time" + +// SnapshotStatsResponse is a representation of the snapshots stats +type SnapshotStatsResponse struct { + Snapshots []SnapshotStatDataResponse `json:"snapshots"` +} + +// SnapshotStatDataResponse is a representation of the single snapshot stat +type SnapshotStatDataResponse struct { + Snapshot string `json:"snapshot"` + UUID string `json:"uuid"` + VersionID int64 `json:"version_id"` + Version string `json:"version"` + Indices []string `json:"indices"` + State string `json:"state"` + StartTime time.Time `json:"start_time"` + StartTimeInMillis int64 `json:"start_time_in_millis"` + EndTime time.Time `json:"end_time"` + EndTimeInMillis int64 `json:"end_time_in_millis"` + DurationInMillis int64 `json:"duration_in_millis"` + Failures []interface{} `json:"failures"` + Shards struct { + Total int64 `json:"total"` + Failed int64 `json:"failed"` + Successful int64 `json:"successful"` + } `json:"shards"` +} + +// SnapshotRepositoriesResponse is a representation snapshots repositories +type SnapshotRepositoriesResponse map[string]struct { + Type string `json:"type"` +} diff --git a/inputs/elasticsearch/collector/snapshots_test.go b/inputs/elasticsearch/collector/snapshots_test.go new file mode 100644 index 00000000..01412190 --- /dev/null +++ b/inputs/elasticsearch/collector/snapshots_test.go @@ -0,0 +1,224 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestSnapshots(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine -Des.path.repo="/tmp" (1.7.6, 2.4.5) + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine -E path.repo="/tmp" (5.4.2) + // curl -XPUT http://localhost:9200/foo_1/type1/1 -d '{"title":"abc","content":"hello"}' + // curl -XPUT http://localhost:9200/foo_1/type1/2 -d '{"title":"def","content":"world"}' + // curl -XPUT http://localhost:9200/foo_2/type1/1 -d '{"title":"abc001","content":"hello001"}' + // curl -XPUT http://localhost:9200/foo_2/type1/2 -d '{"title":"def002","content":"world002"}' + // curl -XPUT http://localhost:9200/foo_2/type1/3 -d '{"title":"def003","content":"world003"}' + // curl -XPUT http://localhost:9200/_snapshot/test1 -d '{"type": "fs","settings":{"location": "/tmp/test1"}}' + // curl -XPUT "http://localhost:9200/_snapshot/test1/snapshot_1?wait_for_completion=true" + // curl http://localhost:9200/_snapshot/ + // curl http://localhost:9200/_snapshot/test1/_all + + tests := []struct { + name string + file string + want string + }{ + { + name: "1.7.6", + file: "../fixtures/snapshots/1.7.6.json", + want: `# HELP elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds Timestamp of the latest SUCCESS or PARTIAL snapshot + # TYPE elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds gauge + elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds{repository="test1"} 1.536052142e+09 + # HELP elasticsearch_snapshot_stats_number_of_snapshots Number of snapshots in a repository + # TYPE elasticsearch_snapshot_stats_number_of_snapshots gauge + elasticsearch_snapshot_stats_number_of_snapshots{repository="test1"} 1 + # HELP elasticsearch_snapshot_stats_oldest_snapshot_timestamp Timestamp of the oldest snapshot + # TYPE elasticsearch_snapshot_stats_oldest_snapshot_timestamp gauge + elasticsearch_snapshot_stats_oldest_snapshot_timestamp{repository="test1"} 1.536052142e+09 + # HELP elasticsearch_snapshot_stats_snapshot_end_time_timestamp Last snapshot end timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_end_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_end_time_timestamp{repository="test1",state="SUCCESS",version="1.7.6"} 1.536052142e+09 + # HELP elasticsearch_snapshot_stats_snapshot_failed_shards Last snapshot failed shards + # TYPE elasticsearch_snapshot_stats_snapshot_failed_shards gauge + elasticsearch_snapshot_stats_snapshot_failed_shards{repository="test1",state="SUCCESS",version="1.7.6"} 0 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_failures Last snapshot number of failures + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_failures gauge + elasticsearch_snapshot_stats_snapshot_number_of_failures{repository="test1",state="SUCCESS",version="1.7.6"} 0 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_indices Number of indices in the last snapshot + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_indices gauge + elasticsearch_snapshot_stats_snapshot_number_of_indices{repository="test1",state="SUCCESS",version="1.7.6"} 2 + # HELP elasticsearch_snapshot_stats_snapshot_start_time_timestamp Last snapshot start timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_start_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_start_time_timestamp{repository="test1",state="SUCCESS",version="1.7.6"} 1.536052142e+09 + # HELP elasticsearch_snapshot_stats_snapshot_successful_shards Last snapshot successful shards + # TYPE elasticsearch_snapshot_stats_snapshot_successful_shards gauge + elasticsearch_snapshot_stats_snapshot_successful_shards{repository="test1",state="SUCCESS",version="1.7.6"} 10 + # HELP elasticsearch_snapshot_stats_snapshot_total_shards Last snapshot total shards + # TYPE elasticsearch_snapshot_stats_snapshot_total_shards gauge + elasticsearch_snapshot_stats_snapshot_total_shards{repository="test1",state="SUCCESS",version="1.7.6"} 10 +`, + }, + { + name: "2.4.5", + file: "../fixtures/snapshots/2.4.5.json", + want: `# HELP elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds Timestamp of the latest SUCCESS or PARTIAL snapshot + # TYPE elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds gauge + elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds{repository="test1"} 1.536053125e+09 + # HELP elasticsearch_snapshot_stats_number_of_snapshots Number of snapshots in a repository + # TYPE elasticsearch_snapshot_stats_number_of_snapshots gauge + elasticsearch_snapshot_stats_number_of_snapshots{repository="test1"} 1 + # HELP elasticsearch_snapshot_stats_oldest_snapshot_timestamp Timestamp of the oldest snapshot + # TYPE elasticsearch_snapshot_stats_oldest_snapshot_timestamp gauge + elasticsearch_snapshot_stats_oldest_snapshot_timestamp{repository="test1"} 1.536053125e+09 + # HELP elasticsearch_snapshot_stats_snapshot_end_time_timestamp Last snapshot end timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_end_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_end_time_timestamp{repository="test1",state="SUCCESS",version="2.4.5"} 1.536053126e+09 + # HELP elasticsearch_snapshot_stats_snapshot_failed_shards Last snapshot failed shards + # TYPE elasticsearch_snapshot_stats_snapshot_failed_shards gauge + elasticsearch_snapshot_stats_snapshot_failed_shards{repository="test1",state="SUCCESS",version="2.4.5"} 0 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_failures Last snapshot number of failures + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_failures gauge + elasticsearch_snapshot_stats_snapshot_number_of_failures{repository="test1",state="SUCCESS",version="2.4.5"} 0 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_indices Number of indices in the last snapshot + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_indices gauge + elasticsearch_snapshot_stats_snapshot_number_of_indices{repository="test1",state="SUCCESS",version="2.4.5"} 2 + # HELP elasticsearch_snapshot_stats_snapshot_start_time_timestamp Last snapshot start timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_start_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_start_time_timestamp{repository="test1",state="SUCCESS",version="2.4.5"} 1.536053125e+09 + # HELP elasticsearch_snapshot_stats_snapshot_successful_shards Last snapshot successful shards + # TYPE elasticsearch_snapshot_stats_snapshot_successful_shards gauge + elasticsearch_snapshot_stats_snapshot_successful_shards{repository="test1",state="SUCCESS",version="2.4.5"} 10 + # HELP elasticsearch_snapshot_stats_snapshot_total_shards Last snapshot total shards + # TYPE elasticsearch_snapshot_stats_snapshot_total_shards gauge + elasticsearch_snapshot_stats_snapshot_total_shards{repository="test1",state="SUCCESS",version="2.4.5"} 10 + `, + }, + { + name: "5.4.2", + file: "../fixtures/snapshots/5.4.2.json", + want: `# HELP elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds Timestamp of the latest SUCCESS or PARTIAL snapshot + # TYPE elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds gauge + elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds{repository="test1"} 1.536053353e+09 + # HELP elasticsearch_snapshot_stats_number_of_snapshots Number of snapshots in a repository + # TYPE elasticsearch_snapshot_stats_number_of_snapshots gauge + elasticsearch_snapshot_stats_number_of_snapshots{repository="test1"} 1 + # HELP elasticsearch_snapshot_stats_oldest_snapshot_timestamp Timestamp of the oldest snapshot + # TYPE elasticsearch_snapshot_stats_oldest_snapshot_timestamp gauge + elasticsearch_snapshot_stats_oldest_snapshot_timestamp{repository="test1"} 1.536053353e+09 + # HELP elasticsearch_snapshot_stats_snapshot_end_time_timestamp Last snapshot end timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_end_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_end_time_timestamp{repository="test1",state="SUCCESS",version="5.4.2"} 1.536053354e+09 + # HELP elasticsearch_snapshot_stats_snapshot_failed_shards Last snapshot failed shards + # TYPE elasticsearch_snapshot_stats_snapshot_failed_shards gauge + elasticsearch_snapshot_stats_snapshot_failed_shards{repository="test1",state="SUCCESS",version="5.4.2"} 0 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_failures Last snapshot number of failures + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_failures gauge + elasticsearch_snapshot_stats_snapshot_number_of_failures{repository="test1",state="SUCCESS",version="5.4.2"} 0 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_indices Number of indices in the last snapshot + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_indices gauge + elasticsearch_snapshot_stats_snapshot_number_of_indices{repository="test1",state="SUCCESS",version="5.4.2"} 2 + # HELP elasticsearch_snapshot_stats_snapshot_start_time_timestamp Last snapshot start timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_start_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_start_time_timestamp{repository="test1",state="SUCCESS",version="5.4.2"} 1.536053353e+09 + # HELP elasticsearch_snapshot_stats_snapshot_successful_shards Last snapshot successful shards + # TYPE elasticsearch_snapshot_stats_snapshot_successful_shards gauge + elasticsearch_snapshot_stats_snapshot_successful_shards{repository="test1",state="SUCCESS",version="5.4.2"} 10 + # HELP elasticsearch_snapshot_stats_snapshot_total_shards Last snapshot total shards + # TYPE elasticsearch_snapshot_stats_snapshot_total_shards gauge + elasticsearch_snapshot_stats_snapshot_total_shards{repository="test1",state="SUCCESS",version="5.4.2"} 10 + `, + }, + { + name: "5.4.2-failure", + file: "../fixtures/snapshots/5.4.2-failed.json", + want: `# HELP elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds Timestamp of the latest SUCCESS or PARTIAL snapshot + # TYPE elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds gauge + elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds{repository="test1"} 1.536053353e+09 + # HELP elasticsearch_snapshot_stats_number_of_snapshots Number of snapshots in a repository + # TYPE elasticsearch_snapshot_stats_number_of_snapshots gauge + elasticsearch_snapshot_stats_number_of_snapshots{repository="test1"} 1 + # HELP elasticsearch_snapshot_stats_oldest_snapshot_timestamp Timestamp of the oldest snapshot + # TYPE elasticsearch_snapshot_stats_oldest_snapshot_timestamp gauge + elasticsearch_snapshot_stats_oldest_snapshot_timestamp{repository="test1"} 1.536053353e+09 + # HELP elasticsearch_snapshot_stats_snapshot_end_time_timestamp Last snapshot end timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_end_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_end_time_timestamp{repository="test1",state="SUCCESS",version="5.4.2"} 1.536053354e+09 + # HELP elasticsearch_snapshot_stats_snapshot_failed_shards Last snapshot failed shards + # TYPE elasticsearch_snapshot_stats_snapshot_failed_shards gauge + elasticsearch_snapshot_stats_snapshot_failed_shards{repository="test1",state="SUCCESS",version="5.4.2"} 1 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_failures Last snapshot number of failures + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_failures gauge + elasticsearch_snapshot_stats_snapshot_number_of_failures{repository="test1",state="SUCCESS",version="5.4.2"} 1 + # HELP elasticsearch_snapshot_stats_snapshot_number_of_indices Number of indices in the last snapshot + # TYPE elasticsearch_snapshot_stats_snapshot_number_of_indices gauge + elasticsearch_snapshot_stats_snapshot_number_of_indices{repository="test1",state="SUCCESS",version="5.4.2"} 2 + # HELP elasticsearch_snapshot_stats_snapshot_start_time_timestamp Last snapshot start timestamp + # TYPE elasticsearch_snapshot_stats_snapshot_start_time_timestamp gauge + elasticsearch_snapshot_stats_snapshot_start_time_timestamp{repository="test1",state="SUCCESS",version="5.4.2"} 1.536053353e+09 + # HELP elasticsearch_snapshot_stats_snapshot_successful_shards Last snapshot successful shards + # TYPE elasticsearch_snapshot_stats_snapshot_successful_shards gauge + elasticsearch_snapshot_stats_snapshot_successful_shards{repository="test1",state="SUCCESS",version="5.4.2"} 10 + # HELP elasticsearch_snapshot_stats_snapshot_total_shards Last snapshot total shards + # TYPE elasticsearch_snapshot_stats_snapshot_total_shards gauge + elasticsearch_snapshot_stats_snapshot_total_shards{repository="test1",state="SUCCESS",version="5.4.2"} 10 + `, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/_snapshot" { + fmt.Fprint(w, `{"test1":{"type":"fs","settings":{"location":"/tmp/test1"}}}`) + return + } + io.Copy(w, f) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + s := NewSnapshots(http.DefaultClient, u) + + // TODO: Convert to collector interface + // c, err := NewSnapshots(log.NewNopLogger(), u, http.DefaultClient) + // if err != nil { + // t.Fatal(err) + // } + + if err := testutil.CollectAndCompare(s, strings.NewReader(tt.want)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/tasks.go b/inputs/elasticsearch/collector/tasks.go new file mode 100644 index 00000000..39f06ecc --- /dev/null +++ b/inputs/elasticsearch/collector/tasks.go @@ -0,0 +1,133 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + + "github.com/alecthomas/kingpin/v2" + "github.com/prometheus/client_golang/prometheus" +) + +// filterByTask global required because collector interface doesn't expose any way to take +// constructor args. +var actionFilter string + +var taskActionDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "task_stats", "action"), + "Number of tasks of a certain action", + []string{"action"}, nil) + +func init() { + kingpin.Flag("tasks.actions", + "Filter on task actions. Used in same way as Task API actions param"). + Default("indices:*").StringVar(&actionFilter) + registerCollector("tasks", defaultDisabled, NewTaskCollector) +} + +// Task Information Struct +type TaskCollector struct { + hc *http.Client + u *url.URL +} + +// NewTaskCollector defines Task Prometheus metrics +func NewTaskCollector(u *url.URL, hc *http.Client) (Collector, error) { + return &TaskCollector{ + hc: hc, + u: u, + }, nil +} + +func (t *TaskCollector) Update(ctx context.Context, ch chan<- prometheus.Metric) error { + tasks, err := t.fetchTasks(ctx) + if err != nil { + return fmt.Errorf("failed to fetch and decode task stats: %w", err) + } + + stats := AggregateTasks(tasks) + for action, count := range stats.CountByAction { + ch <- prometheus.MustNewConstMetric( + taskActionDesc, + prometheus.GaugeValue, + float64(count), + action, + ) + } + return nil +} + +func (t *TaskCollector) fetchTasks(_ context.Context) (tasksResponse, error) { + u := t.u.ResolveReference(&url.URL{Path: "_tasks"}) + q := u.Query() + q.Set("group_by", "none") + q.Set("actions", actionFilter) + u.RawQuery = q.Encode() + + var tr tasksResponse + res, err := t.hc.Get(u.String()) + if err != nil { + return tr, fmt.Errorf("failed to get data stream stats health from %s://%s:%s%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return tr, fmt.Errorf("HTTP Request to %v failed with code %d", u.String(), res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return tr, err + } + + err = json.Unmarshal(bts, &tr) + return tr, err +} + +// tasksResponse is a representation of the Task management API. +type tasksResponse struct { + Tasks []taskResponse `json:"tasks"` +} + +// taskResponse is a representation of the individual task item returned by task API endpoint. +// +// We only parse a very limited amount of this API for use in aggregation. +type taskResponse struct { + Action string `json:"action"` +} + +type AggregatedTaskStats struct { + CountByAction map[string]int64 +} + +func AggregateTasks(t tasksResponse) AggregatedTaskStats { + actions := map[string]int64{} + for _, task := range t.Tasks { + actions[task.Action]++ + } + return AggregatedTaskStats{CountByAction: actions} +} diff --git a/inputs/elasticsearch/collector/tasks_test.go b/inputs/elasticsearch/collector/tasks_test.go new file mode 100644 index 00000000..51af554c --- /dev/null +++ b/inputs/elasticsearch/collector/tasks_test.go @@ -0,0 +1,77 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestTasks(t *testing.T) { + // Test data was collected by running the following: + // # create container + // docker run -d --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.17.11 + // sleep 15 + // # start some busy work in background + // for i in $(seq 1 500) + // do + // curl -o /dev/null -sX POST "localhost:9200/a1/_doc" -H 'Content-Type: application/json' -d'{"a1": "'"$i"'"}' + // sleep .01 + // curl -o /dev/null -sX POST "localhost:9200/a1/_doc" -H 'Content-Type: application/json' -d'{"a2": "'"$i"'"}' + // sleep .01 + // curl -o /dev/null -sX POST "localhost:9200/a1/_doc" -H 'Content-Type: application/json' -d'{"a3": "'"$i"'"}' + // sleep .01 + // done & + // # try and collect a good sample + // curl -X GET 'localhost:9200/_tasks?group_by=none&actions=indices:*' + // # cleanup + // docker rm --force elasticsearch + tcs := map[string]string{ + "7.17": `{"tasks":[{"node":"9lWCm1y_QkujaAg75bVx7A","id":70,"type":"transport","action":"indices:admin/index_template/put","start_time_in_millis":1695900464655,"running_time_in_nanos":308640039,"cancellable":false,"headers":{}},{"node":"9lWCm1y_QkujaAg75bVx7A","id":73,"type":"transport","action":"indices:admin/index_template/put","start_time_in_millis":1695900464683,"running_time_in_nanos":280672000,"cancellable":false,"headers":{}},{"node":"9lWCm1y_QkujaAg75bVx7A","id":76,"type":"transport","action":"indices:admin/index_template/put","start_time_in_millis":1695900464711,"running_time_in_nanos":253247906,"cancellable":false,"headers":{}},{"node":"9lWCm1y_QkujaAg75bVx7A","id":93,"type":"transport","action":"indices:admin/index_template/put","start_time_in_millis":1695900464904,"running_time_in_nanos":60230460,"cancellable":false,"headers":{}},{"node":"9lWCm1y_QkujaAg75bVx7A","id":50,"type":"transport","action":"indices:data/write/index","start_time_in_millis":1695900464229,"running_time_in_nanos":734480468,"cancellable":false,"headers":{}},{"node":"9lWCm1y_QkujaAg75bVx7A","id":51,"type":"transport","action":"indices:admin/auto_create","start_time_in_millis":1695900464235,"running_time_in_nanos":729223933,"cancellable":false,"headers":{}}]}`, + } + want := `# HELP elasticsearch_task_stats_action Number of tasks of a certain action +# TYPE elasticsearch_task_stats_action gauge +elasticsearch_task_stats_action{action="indices:admin/auto_create"} 1 +elasticsearch_task_stats_action{action="indices:admin/index_template/put"} 4 +elasticsearch_task_stats_action{action="indices:data/write/index"} 1 +` + for ver, out := range tcs { + t.Run(ver, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprintln(w, out) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + + c, err := NewTaskCollector(u, ts.Client()) + if err != nil { + t.Fatalf("Failed to create collector: %v", err) + } + + if err := testutil.CollectAndCompare(wrapCollector{c}, strings.NewReader(want)); err != nil { + t.Fatalf("Metrics did not match: %v", err) + } + }) + } +} diff --git a/inputs/elasticsearch/collector/versions_test.go b/inputs/elasticsearch/collector/versions_test.go new file mode 100644 index 00000000..2158cf59 --- /dev/null +++ b/inputs/elasticsearch/collector/versions_test.go @@ -0,0 +1,24 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +var testElasticsearchVersions = []string{ + "5.4.2", + "5.6.16", + "6.5.4", + "6.8.8", + "7.3.0", + "7.6.2", + "7.13.1", +} diff --git a/inputs/elasticsearch/elasticsearch.go b/inputs/elasticsearch/elasticsearch.go index d37bdd5e..fb247309 100644 --- a/inputs/elasticsearch/elasticsearch.go +++ b/inputs/elasticsearch/elasticsearch.go @@ -1,94 +1,90 @@ package elasticsearch import ( - "encoding/json" + "context" + "errors" "fmt" - "io" "log" "net/http" - "sort" - "strings" + "net/url" + "os" + "os/signal" "sync" "time" "flashcat.cloud/categraf/config" "flashcat.cloud/categraf/inputs" + "flashcat.cloud/categraf/inputs/elasticsearch/collector" + "flashcat.cloud/categraf/inputs/elasticsearch/pkg/clusterinfo" + "flashcat.cloud/categraf/inputs/elasticsearch/pkg/roundtripper" "flashcat.cloud/categraf/pkg/filter" - "flashcat.cloud/categraf/pkg/jsonx" "flashcat.cloud/categraf/pkg/tls" "flashcat.cloud/categraf/types" + + "github.com/prometheus/common/version" ) const inputName = "elasticsearch" -// Nodestats are always generated, so simply define a constant for these endpoints -const statsPath = "/_nodes/stats" -const statsPathLocal = "/_nodes/_local/stats" - -type nodeStat struct { - Host string `json:"host"` - Name string `json:"name"` - Roles []string `json:"roles"` - Attributes map[string]string `json:"attributes"` - Indices interface{} `json:"indices"` - OS interface{} `json:"os"` - Process interface{} `json:"process"` - JVM interface{} `json:"jvm"` - ThreadPool interface{} `json:"thread_pool"` - FS interface{} `json:"fs"` - Transport interface{} `json:"transport"` - HTTP interface{} `json:"http"` - Breakers interface{} `json:"breakers"` -} - -type clusterHealth struct { - ActivePrimaryShards int `json:"active_primary_shards"` - ActiveShards int `json:"active_shards"` - ActiveShardsPercentAsNumber float64 `json:"active_shards_percent_as_number"` - ClusterName string `json:"cluster_name"` - DelayedUnassignedShards int `json:"delayed_unassigned_shards"` - InitializingShards int `json:"initializing_shards"` - NumberOfDataNodes int `json:"number_of_data_nodes"` - NumberOfInFlightFetch int `json:"number_of_in_flight_fetch"` - NumberOfNodes int `json:"number_of_nodes"` - NumberOfPendingTasks int `json:"number_of_pending_tasks"` - RelocatingShards int `json:"relocating_shards"` - Status string `json:"status"` - TaskMaxWaitingInQueueMillis int `json:"task_max_waiting_in_queue_millis"` - TimedOut bool `json:"timed_out"` - UnassignedShards int `json:"unassigned_shards"` - Indices map[string]indexHealth `json:"indices"` -} - -type indexHealth struct { - ActivePrimaryShards int `json:"active_primary_shards"` - ActiveShards int `json:"active_shards"` - InitializingShards int `json:"initializing_shards"` - NumberOfReplicas int `json:"number_of_replicas"` - NumberOfShards int `json:"number_of_shards"` - RelocatingShards int `json:"relocating_shards"` - Status string `json:"status"` - UnassignedShards int `json:"unassigned_shards"` -} - -type clusterStats struct { - NodeName string `json:"node_name"` - ClusterName string `json:"cluster_name"` - Status string `json:"status"` - Indices interface{} `json:"indices"` - Nodes interface{} `json:"nodes"` -} - -type indexStat struct { - Primaries interface{} `json:"primaries"` - Total interface{} `json:"total"` - Shards map[string][]interface{} `json:"shards"` -} - -type Elasticsearch struct { - config.PluginConfig - Instances []*Instance `toml:"instances"` -} +var _ inputs.SampleGatherer = new(Instance) +var _ inputs.Input = new(Elasticsearch) +var _ inputs.InstancesGetter = new(Elasticsearch) + +type ( + Elasticsearch struct { + config.PluginConfig + + Instances []*Instance `toml:"instances"` + } + + Instance struct { + config.InstanceConfig + + Local bool `toml:"local"` + Servers []string `toml:"servers"` + UserName string `toml:"username"` + Password string `toml:"password"` + ApiKey string `toml:"api_key"` + HTTPTimeout config.Duration `toml:"http_timeout"` + AllNodes bool `toml:"all_nodes"` + Node string `toml:"node"` + NodeStats []string `toml:"node_stats"` + ClusterHealth bool `toml:"cluster_health"` + ClusterHealthLevel string `toml:"cluster_health_level"` + ClusterStats bool `toml:"cluster_stats"` + IndicesInclude []string `toml:"indices_include"` + ExportIndices bool `toml:"export_indices"` + ExportIndicesSettings bool `toml:"export_indices_settings"` + ExportIndicesMappings bool `toml:"export_indices_mappings"` + ExportIndexAliases bool `toml:"export_index_aliases"` + ExportILM bool `toml:"export_ilm"` + ExportShards bool `toml:"export_shards"` + ExportSLM bool `toml:"export_slm"` + ExportDataStream bool `toml:"export_data_stream"` + ExportSnapshots bool `toml:"export_snapshots"` + ExportClusterSettings bool `toml:"export_cluster_settings"` + ClusterInfoInterval config.Duration `toml:"cluster_info_interval"` + AwsRegion string `toml:"aws_region"` + AwsRoleArn string `toml:"aws_role_arn"` + + EsURL *url.URL + *http.Client + tls.ClientConfig + indexMatchers map[string]filter.Filter + serverInfo map[string]serverInfo + serverInfoMutex sync.Mutex + } + + transportWithAPIKey struct { + underlyingTransport http.RoundTripper + apiKey string + } + + serverInfo struct { + nodeID string + masterID string + } +) func init() { inputs.Add(inputName, func() inputs.Input { @@ -112,49 +108,24 @@ func (r *Elasticsearch) GetInstances() []inputs.Instance { return ret } -type Instance struct { - config.InstanceConfig - - Local bool `toml:"local"` - Servers []string `toml:"servers"` - HTTPTimeout config.Duration `toml:"http_timeout"` - ClusterHealth bool `toml:"cluster_health"` - ClusterHealthLevel string `toml:"cluster_health_level"` - ClusterStats bool `toml:"cluster_stats"` - IndicesInclude []string `toml:"indices_include"` - IndicesLevel string `toml:"indices_level"` - NodeStats []string `toml:"node_stats"` - Username string `toml:"username"` - Password string `toml:"password"` - NumMostRecentIndices int `toml:"num_most_recent_indices"` - - tls.ClientConfig - client *http.Client - indexMatchers map[string]filter.Filter - serverInfo map[string]serverInfo - serverInfoMutex sync.Mutex -} - -type serverInfo struct { - nodeID string - masterID string -} - -func (i serverInfo) isMaster() bool { - return i.nodeID == i.masterID -} - func (ins *Instance) Init() error { if len(ins.Servers) == 0 { return types.ErrInstancesEmpty } - if ins.HTTPTimeout <= 0 { - ins.HTTPTimeout = config.Duration(time.Second * 5) + ins.HTTPTimeout = config.Duration(5 * time.Second) } - - if ins.ClusterHealthLevel == "" { - ins.ClusterHealthLevel = "indices" + if ins.ClusterInfoInterval == 0 { + ins.ClusterInfoInterval = config.Duration(5 * time.Minute) + } + if ins.UserName == "" { + ins.UserName = os.Getenv("ES_USERNAME") + } + if ins.Password == "" { + ins.Password = os.Getenv("ES_PASSWORD") + } + if ins.ApiKey == "" { + ins.ApiKey = os.Getenv("ES_API_KEY") } // Compile the configured indexes to match for sorting. @@ -162,31 +133,22 @@ func (ins *Instance) Init() error { if err != nil { return err } - ins.indexMatchers = indexMatchers - ins.client, err = ins.createHTTPClient() - return err -} -func (ins *Instance) compileIndexMatchers() (map[string]filter.Filter, error) { - indexMatchers := map[string]filter.Filter{} - var err error - - // Compile each configured index into a glob matcher. - for _, configuredIndex := range ins.IndicesInclude { - if _, exists := indexMatchers[configuredIndex]; !exists { - indexMatchers[configuredIndex], err = filter.Compile([]string{configuredIndex}) - if err != nil { - return nil, err - } - } + ins.Client, err = ins.createHTTPClient() + if err != nil { + return err } - return indexMatchers, nil + return nil } func (ins *Instance) Gather(slist *types.SampleList) { - if ins.ClusterStats || len(ins.IndicesInclude) > 0 || len(ins.IndicesLevel) > 0 { + // version metric + if err := inputs.Collect(version.NewCollector(inputName), slist); err != nil { + log.Println("E! failed to collect version metric:", err) + } + if ins.ClusterStats || len(ins.IndicesInclude) > 0 { var wgC sync.WaitGroup wgC.Add(len(ins.Servers)) @@ -195,11 +157,10 @@ func (ins *Instance) Gather(slist *types.SampleList) { go func(s string, slist *types.SampleList) { defer wgC.Done() info := serverInfo{} - var err error // Gather node ID - if info.nodeID, err = ins.gatherNodeID(s + "/_nodes/_local/name"); err != nil { + if info.nodeID, err = collector.GetNodeID(ins.Client, ins.UserName, ins.Password, s); err != nil { slist.PushSample("elasticsearch", "up", 0, map[string]string{"address": s}) log.Println("E! failed to gather node id:", err) return @@ -207,7 +168,7 @@ func (ins *Instance) Gather(slist *types.SampleList) { // get cat/master information here so NodeStats can determine // whether this node is the Master - if info.masterID, err = ins.getCatMaster(s + "/_cat/master"); err != nil { + if info.masterID, err = collector.GetCatMaster(ins.Client, ins.UserName, ins.Password, s); err != nil { slist.PushSample("elasticsearch", "up", 0, map[string]string{"address": s}) log.Println("E! failed to get cat master:", err) return @@ -225,482 +186,210 @@ func (ins *Instance) Gather(slist *types.SampleList) { var wg sync.WaitGroup wg.Add(len(ins.Servers)) + // create the exporter for _, serv := range ins.Servers { go func(s string, slist *types.SampleList) { defer wg.Done() - url := ins.nodeStatsURL(s) + EsUrl, err := url.Parse(s) + if err != nil { + log.Println("failed to parse es_uri, err: ", err) + return + } + if ins.UserName != "" && ins.Password != "" { + EsUrl.User = url.UserPassword(ins.UserName, ins.Password) + } + exporter, err := collector.NewElasticsearchCollector( + []string{}, + collector.WithElasticsearchURL(EsUrl), + collector.WithHTTPClient(ins.Client), + ) + if err != nil { + log.Println("E! failed to create Elasticsearch collector, err: ", err) + return + } + if err := inputs.Collect(exporter, slist); err != nil { + log.Println("E! failed to collect metrics:", err) + } // Always gather node stats - if err := ins.gatherNodeStats(url, s, slist); err != nil { - log.Println("E! failed to gather node stats:", err) - return + if err := inputs.Collect(collector.NewNodes(ins.Client, EsUrl, ins.AllNodes, ins.Node, ins.Local, ins.NodeStats), slist); err != nil { + log.Println("E! failed to collect nodes metrics:", err) } + clusterInfoRetriever := clusterinfo.New(ins.Client, EsUrl, time.Duration(ins.ClusterInfoInterval)) + if ins.ClusterHealth { - url = s + "/_cluster/health" - if ins.ClusterHealthLevel != "" { - url = url + "?level=" + ins.ClusterHealthLevel - } - if err := ins.gatherClusterHealth(url, s, slist); err != nil { - log.Println("E! failed to gather cluster health:", err) - return + if ins.ClusterHealthLevel == "indices" { + if err := inputs.Collect(collector.NewClusterHealthIndices(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect cluster health indices metrics:", err) + } + } else { + if err := inputs.Collect(collector.NewClusterHealth(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect cluster health metrics:", err) + } } } if ins.ClusterStats && (ins.serverInfo[s].isMaster() || !ins.Local) { - if err := ins.gatherClusterStats(s+"/_cluster/stats", s, slist); err != nil { - log.Println("E! failed to gather cluster stats:", err) - return + if err := inputs.Collect(collector.NewClusterStats(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect cluster stats metrics:", err) } } - if len(ins.IndicesInclude) > 0 && (ins.serverInfo[s].isMaster() || !ins.Local) { - if ins.IndicesLevel != "shards" { - if err := ins.gatherIndicesStats(s+"/"+strings.Join(ins.IndicesInclude, ",")+"/_stats", s, slist); err != nil { - log.Println("E! failed to gather indices stats:", err) - return - } - } else { - if err := ins.gatherIndicesStats(s+"/"+strings.Join(ins.IndicesInclude, ",")+"/_stats?level=shards", s, slist); err != nil { - log.Println("E! failed to gather indices stats:", err) - return - } + if (ins.ExportIndices || ins.ExportShards) && (ins.serverInfo[s].isMaster() || !ins.Local) { + sC := collector.NewShards(ins.Client, EsUrl) + if err := inputs.Collect(sC, slist); err != nil { + log.Println("E! failed to collect shards metrics:", err) + } + iC := collector.NewIndices(ins.Client, EsUrl, ins.ExportShards, ins.ExportIndexAliases, ins.IndicesInclude) + if err := inputs.Collect(iC, slist); err != nil { + log.Println("E! failed to collect indices metrics:", err) + } + if registerErr := clusterInfoRetriever.RegisterConsumer(iC); registerErr != nil { + log.Println("failed to register indices collector in cluster info") + } + if registerErr := clusterInfoRetriever.RegisterConsumer(sC); registerErr != nil { + log.Println("failed to register shards collector in cluster info") } } - }(serv, slist) - } - - wg.Wait() -} - -func (ins *Instance) gatherIndicesStats(url string, address string, slist *types.SampleList) error { - indicesStats := &struct { - Shards map[string]interface{} `json:"_shards"` - All map[string]interface{} `json:"_all"` - Indices map[string]indexStat `json:"indices"` - }{} - - if err := ins.gatherJSONData(url, indicesStats); err != nil { - return err - } - - addrTag := map[string]string{"address": address} - - // Total Shards Stats - slist.PushSamples("elasticsearch_indices_stats_shards_total", indicesStats.Shards, addrTag) - - // All Stats - for m, s := range indicesStats.All { - // parse Json, ignoring bools and excluding strings - jsonParser := jsonx.JSONFlattener{} - err := jsonParser.FullFlattenJSON("_", s, false, true) - if err != nil { - return err - } - for key, val := range jsonParser.Fields { - slist.PushSample("elasticsearch", "indices_stats_"+m+"_"+key, val, map[string]string{"index_name": "_all"}, addrTag) - } - } - // Gather stats for each index. - return ins.gatherIndividualIndicesStats(indicesStats.Indices, addrTag, slist) -} - -// gatherSortedIndicesStats gathers stats for all indices in no particular order. -func (ins *Instance) gatherIndividualIndicesStats(indices map[string]indexStat, addrTag map[string]string, slist *types.SampleList) error { - // Sort indices into buckets based on their configured prefix, if any matches. - categorizedIndexNames := ins.categorizeIndices(indices) - for _, matchingIndices := range categorizedIndexNames { - // Establish the number of each category of indices to use. User can configure to use only the latest 'X' amount. - indicesCount := len(matchingIndices) - indicesToTrackCount := indicesCount - - // Sort the indices if configured to do so. - if ins.NumMostRecentIndices > 0 { - if ins.NumMostRecentIndices < indicesToTrackCount { - indicesToTrackCount = ins.NumMostRecentIndices + if ins.ExportSLM { + if err := inputs.Collect(collector.NewSLM(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect SLM metrics:", err) + } } - sort.Strings(matchingIndices) - } - // Gather only the number of indexes that have been configured, in descending order (most recent, if date-stamped). - for i := indicesCount - 1; i >= indicesCount-indicesToTrackCount; i-- { - indexName := matchingIndices[i] - - err := ins.gatherSingleIndexStats(indexName, indices[indexName], addrTag, slist) - if err != nil { - return err + if ins.ExportDataStream { + if err := inputs.Collect(collector.NewDataStream(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect data stream metrics:", err) + } } - } - } - - return nil -} - -func (ins *Instance) gatherSingleIndexStats(name string, index indexStat, addrTag map[string]string, slist *types.SampleList) error { - indexTag := map[string]string{"index_name": name} - stats := map[string]interface{}{ - "primaries": index.Primaries, - "total": index.Total, - } - for m, s := range stats { - f := jsonx.JSONFlattener{} - // parse Json, getting strings and bools - err := f.FullFlattenJSON("", s, true, true) - if err != nil { - return err - } - for key, val := range f.Fields { - slist.PushSample("elasticsearch", "indices_stats_"+m+"_"+key, val, indexTag, addrTag) - } - } - if ins.IndicesLevel == "shards" { - for shardNumber, shards := range index.Shards { - for _, shard := range shards { - // Get Shard Stats - flattened := jsonx.JSONFlattener{} - err := flattened.FullFlattenJSON("", shard, true, true) - if err != nil { - return err + if ins.ExportIndicesSettings { + if err := inputs.Collect(collector.NewIndicesSettings(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect indices settings metrics:", err) } + } - // determine shard tag and primary/replica designation - shardType := "replica" - routingPrimary, _ := flattened.Fields["routing_primary"].(bool) - if routingPrimary { - shardType = "primary" + if ins.ExportIndicesMappings { + if err := inputs.Collect(collector.NewIndicesMappings(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect indices mappings metrics:", err) } - delete(flattened.Fields, "routing_primary") + } - routingState, ok := flattened.Fields["routing_state"].(string) - if ok { - flattened.Fields["routing_state"] = mapShardStatusToCode(routingState) + if ins.ExportSnapshots { + if err := inputs.Collect(collector.NewSnapshots(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect snapshot metrics:", err) } + } - routingNode, _ := flattened.Fields["routing_node"].(string) - shardTags := map[string]string{ - "index_name": name, - "node_id": routingNode, - "shard_name": shardNumber, - "type": shardType, + if ins.ExportILM { + if err := inputs.Collect(collector.NewIlmStatus(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect ilm status metrics:", err) } - - for key, field := range flattened.Fields { - switch field.(type) { - case string, bool: - delete(flattened.Fields, key) - } + if err := inputs.Collect(collector.NewIlmIndicies(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect ilm indices metrics:", err) } - - slist.PushSamples("elasticsearch_indices_stats_shards", flattened.Fields, shardTags, addrTag) } - } - } - - return nil -} - -func (ins *Instance) categorizeIndices(indices map[string]indexStat) map[string][]string { - categorizedIndexNames := map[string][]string{} - // If all indices are configured to be gathered, bucket them all together. - if len(ins.IndicesInclude) == 0 || ins.IndicesInclude[0] == "_all" { - for indexName := range indices { - categorizedIndexNames["_all"] = append(categorizedIndexNames["_all"], indexName) - } + if ins.ExportClusterSettings { + if err := inputs.Collect(collector.NewClusterSettings(ins.Client, EsUrl), slist); err != nil { + log.Println("E! failed to collect cluster settings metrics:", err) + } + } - return categorizedIndexNames - } + // Create a context that is cancelled on SIGKILL or SIGINT. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) + defer cancel() - // Bucket each returned index with its associated configured index (if any match). - for indexName := range indices { - match := indexName - for name, matcher := range ins.indexMatchers { - // If a configured index matches one of the returned indexes, mark it as a match. - if matcher.Match(match) { - match = name - break + // start the cluster info retriever + switch runErr := clusterInfoRetriever.Run(ctx); { + case runErr == nil: + if ins.DebugMod { + log.Println("started cluster info retriever, interval: ", ins.ClusterInfoInterval) + } + case errors.Is(runErr, clusterinfo.ErrInitialCallTimeout): + if ins.DebugMod { + log.Println("initial cluster info call timed out") + } + default: + log.Println("failed to run cluster info retriever, err: ", err) + return } - } - // Bucket all matching indices together for sorting. - categorizedIndexNames[match] = append(categorizedIndexNames[match], indexName) + // register cluster info retriever as prometheus collector + if err := inputs.Collect(clusterInfoRetriever, slist); err != nil { + log.Println("E! failed to collect cluster info metrics:", err) + } + }(serv, slist) } - return categorizedIndexNames + wg.Wait() + return } -func (ins *Instance) gatherClusterStats(url string, address string, slist *types.SampleList) error { - clusterStats := &clusterStats{} - if err := ins.gatherJSONData(url, clusterStats); err != nil { - return err - } - - tags := map[string]string{ - // "node_name": clusterStats.NodeName, - // "status": clusterStats.Status, - "cluster_name": clusterStats.ClusterName, - "address": address, - } - - stats := map[string]interface{}{ - "nodes": clusterStats.Nodes, - "indices": clusterStats.Indices, +func (ins *Instance) createHTTPClient() (*http.Client, error) { + var httpTransport http.RoundTripper + var err error + httpTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConnsPerHost: 1, + } + if ins.ApiKey != "" { + httpTransport = &transportWithAPIKey{ + underlyingTransport: httpTransport, + apiKey: ins.ApiKey, + } } - for p, s := range stats { - f := jsonx.JSONFlattener{} - // parse json, including bools and excluding strings - err := f.FullFlattenJSON("", s, false, true) + if ins.UseTLS { + tlsConfig, err := ins.ClientConfig.TLSConfig() if err != nil { - return err + return nil, err } - - for key, val := range f.Fields { - slist.PushSample("elasticsearch", "clusterstats_"+p+"_"+key, val, tags) + httpTransport = &http.Transport{ + TLSClientConfig: tlsConfig, + Proxy: http.ProxyFromEnvironment, + MaxIdleConnsPerHost: 1, } } - return nil -} - -func (ins *Instance) gatherClusterHealth(url string, address string, slist *types.SampleList) error { - healthStats := &clusterHealth{} - if err := ins.gatherJSONData(url, healthStats); err != nil { - return err - } - - addrTag := map[string]string{"address": address} - - clusterFields := map[string]interface{}{ - "cluster_health_active_primary_shards": healthStats.ActivePrimaryShards, - "cluster_health_active_shards": healthStats.ActiveShards, - "cluster_health_active_shards_percent_as_number": healthStats.ActiveShardsPercentAsNumber, - "cluster_health_delayed_unassigned_shards": healthStats.DelayedUnassignedShards, - "cluster_health_initializing_shards": healthStats.InitializingShards, - "cluster_health_number_of_data_nodes": healthStats.NumberOfDataNodes, - "cluster_health_number_of_in_flight_fetch": healthStats.NumberOfInFlightFetch, - "cluster_health_number_of_nodes": healthStats.NumberOfNodes, - "cluster_health_number_of_pending_tasks": healthStats.NumberOfPendingTasks, - "cluster_health_relocating_shards": healthStats.RelocatingShards, - "cluster_health_status_code": mapHealthStatusToCode(healthStats.Status), - "cluster_health_task_max_waiting_in_queue_millis": healthStats.TaskMaxWaitingInQueueMillis, - "cluster_health_timed_out": healthStats.TimedOut, - "cluster_health_unassigned_shards": healthStats.UnassignedShards, + client := &http.Client{ + Timeout: time.Duration(ins.HTTPTimeout), + Transport: httpTransport, } - - slist.PushSamples("elasticsearch", clusterFields, map[string]string{"cluster_name": healthStats.ClusterName}, addrTag) - - for name, health := range healthStats.Indices { - indexFields := map[string]interface{}{ - "cluster_health_indices_active_primary_shards": health.ActivePrimaryShards, - "cluster_health_indices_active_shards": health.ActiveShards, - "cluster_health_indices_initializing_shards": health.InitializingShards, - "cluster_health_indices_number_of_replicas": health.NumberOfReplicas, - "cluster_health_indices_number_of_shards": health.NumberOfShards, - "cluster_health_indices_relocating_shards": health.RelocatingShards, - "cluster_health_indices_status_code": mapHealthStatusToCode(health.Status), - "cluster_health_indices_unassigned_shards": health.UnassignedShards, + if ins.AwsRegion != "" { + ins.Client.Transport, err = roundtripper.NewAWSSigningTransport(httpTransport, ins.AwsRegion, ins.AwsRoleArn) + if err != nil { + log.Println("E! failed to create AWS transport, err: ", err) } - slist.PushSamples("elasticsearch", indexFields, map[string]string{"index": name, "name": healthStats.ClusterName}, addrTag) } - return nil + return client, nil } -func (ins *Instance) gatherNodeStats(url string, address string, slist *types.SampleList) error { - nodeStats := &struct { - ClusterName string `json:"cluster_name"` - Nodes map[string]*nodeStat `json:"nodes"` - }{} - - if err := ins.gatherJSONData(url, nodeStats); err != nil { - return err - } - - addrTag := map[string]string{"address": address} - - for id, n := range nodeStats.Nodes { - // sort.Strings(n.Roles) - tags := map[string]string{ - "node_id": id, - "node_host": n.Host, - "node_name": n.Name, - "cluster_name": nodeStats.ClusterName, - // "node_roles": strings.Join(n.Roles, ","), - } - - for k, v := range n.Attributes { - slist.PushSample("elasticsearch", "node_attribute_"+k, v, tags, addrTag) - } - - stats := map[string]interface{}{ - "indices": n.Indices, - "os": n.OS, - "process": n.Process, - "jvm": n.JVM, - "thread_pool": n.ThreadPool, - "fs": n.FS, - "transport": n.Transport, - "http": n.HTTP, - "breakers": n.Breakers, - } +func (ins *Instance) compileIndexMatchers() (map[string]filter.Filter, error) { + indexMatchers := map[string]filter.Filter{} + var err error - for p, s := range stats { - // if one of the individual node stats is not even in the - // original result - if s == nil { - continue - } - f := jsonx.JSONFlattener{} - // parse Json, ignoring strings and bools - err := f.FlattenJSON("", s) + // Compile each configured index into a glob matcher. + for _, configuredIndex := range ins.IndicesInclude { + if _, exists := indexMatchers[configuredIndex]; !exists { + indexMatchers[configuredIndex], err = filter.Compile([]string{configuredIndex}) if err != nil { - return err - } - - for key, val := range f.Fields { - slist.PushSample("elasticsearch", p+"_"+key, val, tags, addrTag) + return nil, err } } } - return nil -} - -func (ins *Instance) nodeStatsURL(baseURL string) string { - var url string - - if ins.Local { - url = baseURL + statsPathLocal - } else { - url = baseURL + statsPath - } - - if len(ins.NodeStats) == 0 { - return url - } - - return fmt.Sprintf("%s/%s", url, strings.Join(ins.NodeStats, ",")) -} - -func (ins *Instance) getCatMaster(url string) (string, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", err - } - - if ins.Username != "" || ins.Password != "" { - req.SetBasicAuth(ins.Username, ins.Password) - } - - r, err := ins.client.Do(req) - if err != nil { - return "", err - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - // NOTE: we are not going to read/discard r.Body under the assumption we'd prefer - // to let the underlying transport close the connection and re-establish a new one for - // future calls. - return "", fmt.Errorf("elasticsearch: Unable to retrieve master node information. API responded with status-code %d, expected %d", r.StatusCode, http.StatusOK) - } - response, err := io.ReadAll(r.Body) - - if err != nil { - return "", err - } - - masterID := strings.Split(string(response), " ")[0] - - return masterID, nil -} - -func (ins *Instance) gatherNodeID(url string) (string, error) { - nodeStats := &struct { - ClusterName string `json:"cluster_name"` - Nodes map[string]*nodeStat `json:"nodes"` - }{} - if err := ins.gatherJSONData(url, nodeStats); err != nil { - return "", err - } - - // Only 1 should be returned - for id := range nodeStats.Nodes { - return id, nil - } - return "", nil -} - -func (ins *Instance) gatherJSONData(url string, v interface{}) error { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - - if ins.Username != "" || ins.Password != "" { - req.SetBasicAuth(ins.Username, ins.Password) - } - - r, err := ins.client.Do(req) - if err != nil { - return err - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - // NOTE: we are not going to read/discard r.Body under the assumption we'd prefer - // to let the underlying transport close the connection and re-establish a new one for - // future calls. - return fmt.Errorf("elasticsearch: API responded with status-code %d, expected %d", - r.StatusCode, http.StatusOK) - } - - return json.NewDecoder(r.Body).Decode(v) -} - -// perform status mapping -func mapHealthStatusToCode(s string) int { - switch strings.ToLower(s) { - case "green": - return 1 - case "yellow": - return 2 - case "red": - return 3 - } - return 0 + return indexMatchers, nil } -// perform shard status mapping -func mapShardStatusToCode(s string) int { - switch strings.ToUpper(s) { - case "UNASSIGNED": - return 1 - case "INITIALIZING": - return 2 - case "STARTED": - return 3 - case "RELOCATING": - return 4 - } - return 0 +func (t *transportWithAPIKey) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Authorization", fmt.Sprintf("ApiKey %s", t.apiKey)) + return t.underlyingTransport.RoundTrip(req) } -func (ins *Instance) createHTTPClient() (*http.Client, error) { - tlsCfg, err := ins.ClientConfig.TLSConfig() - if err != nil { - return nil, err - } - tr := &http.Transport{ - ResponseHeaderTimeout: time.Duration(ins.HTTPTimeout), - TLSClientConfig: tlsCfg, - } - - client := &http.Client{ - Transport: tr, - Timeout: time.Duration(ins.HTTPTimeout), - } - - return client, nil +func (i serverInfo) isMaster() bool { + return i.nodeID == i.masterID } diff --git a/inputs/elasticsearch/fixtures/clusterhealth/1.7.6.json b/inputs/elasticsearch/fixtures/clusterhealth/1.7.6.json new file mode 100644 index 00000000..29ef9cf6 --- /dev/null +++ b/inputs/elasticsearch/fixtures/clusterhealth/1.7.6.json @@ -0,0 +1,15 @@ +{ + "active_primary_shards": 5, + "active_shards": 5, + "cluster_name": "elasticsearch", + "delayed_unassigned_shards": 0, + "initializing_shards": 0, + "number_of_data_nodes": 1, + "number_of_in_flight_fetch": 0, + "number_of_nodes": 1, + "number_of_pending_tasks": 0, + "relocating_shards": 0, + "status": "yellow", + "timed_out": false, + "unassigned_shards": 5 +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/clusterhealth/2.4.5.json b/inputs/elasticsearch/fixtures/clusterhealth/2.4.5.json new file mode 100644 index 00000000..410ece76 --- /dev/null +++ b/inputs/elasticsearch/fixtures/clusterhealth/2.4.5.json @@ -0,0 +1,17 @@ +{ + "active_primary_shards": 5, + "active_shards": 5, + "active_shards_percent_as_number": 50, + "cluster_name": "elasticsearch", + "delayed_unassigned_shards": 0, + "initializing_shards": 0, + "number_of_data_nodes": 1, + "number_of_in_flight_fetch": 0, + "number_of_nodes": 1, + "number_of_pending_tasks": 0, + "relocating_shards": 0, + "status": "yellow", + "task_max_waiting_in_queue_millis": 12, + "timed_out": false, + "unassigned_shards": 5 +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/clusterhealth/5.4.2.json b/inputs/elasticsearch/fixtures/clusterhealth/5.4.2.json new file mode 100644 index 00000000..410ece76 --- /dev/null +++ b/inputs/elasticsearch/fixtures/clusterhealth/5.4.2.json @@ -0,0 +1,17 @@ +{ + "active_primary_shards": 5, + "active_shards": 5, + "active_shards_percent_as_number": 50, + "cluster_name": "elasticsearch", + "delayed_unassigned_shards": 0, + "initializing_shards": 0, + "number_of_data_nodes": 1, + "number_of_in_flight_fetch": 0, + "number_of_nodes": 1, + "number_of_pending_tasks": 0, + "relocating_shards": 0, + "status": "yellow", + "task_max_waiting_in_queue_millis": 12, + "timed_out": false, + "unassigned_shards": 5 +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/clusterinfo/2.4.5.json b/inputs/elasticsearch/fixtures/clusterinfo/2.4.5.json new file mode 100644 index 00000000..14467d3f --- /dev/null +++ b/inputs/elasticsearch/fixtures/clusterinfo/2.4.5.json @@ -0,0 +1,13 @@ +{ + "cluster_name": "elasticsearch", + "cluster_uuid": "3qps7bcWTqyzV49ApmPVfw", + "name": "Mys-Tech", + "tagline": "You Know, for Search", + "version": { + "build_hash": "c849dd13904f53e63e88efc33b2ceeda0b6a1276", + "build_snapshot": false, + "build_timestamp": "2017-04-24T16:18:17Z", + "lucene_version": "5.5.4", + "number": "2.4.5" + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/clusterinfo/5.4.2.json b/inputs/elasticsearch/fixtures/clusterinfo/5.4.2.json new file mode 100644 index 00000000..adc31698 --- /dev/null +++ b/inputs/elasticsearch/fixtures/clusterinfo/5.4.2.json @@ -0,0 +1,13 @@ +{ + "cluster_name": "elasticsearch", + "cluster_uuid": "kbqi7yhQT-WlPdGL2m0xJg", + "name": "gOHPUga", + "tagline": "You Know, for Search", + "version": { + "build_date": "2017-06-15T02:29:28.122Z", + "build_hash": "929b078", + "build_snapshot": false, + "lucene_version": "6.5.1", + "number": "5.4.2" + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/clusterinfo/7.13.1.json b/inputs/elasticsearch/fixtures/clusterinfo/7.13.1.json new file mode 100644 index 00000000..fb2e06ab --- /dev/null +++ b/inputs/elasticsearch/fixtures/clusterinfo/7.13.1.json @@ -0,0 +1,17 @@ +{ + "cluster_name": "docker-cluster", + "cluster_uuid": "aCMrCY1VQpqJ6U4Sw_xdiw", + "name": "e0630cfd8e1e", + "tagline": "You Know, for Search", + "version": { + "build_date": "2021-05-28T17:40:59.346932922Z", + "build_flavor": "default", + "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9", + "build_snapshot": false, + "build_type": "docker", + "lucene_version": "8.8.2", + "minimum_index_compatibility_version": "6.0.0-beta1", + "minimum_wire_compatibility_version": "6.8.0", + "number": "7.13.1" + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/datastream/7.15.0.json b/inputs/elasticsearch/fixtures/datastream/7.15.0.json new file mode 100644 index 00000000..b1ddce8c --- /dev/null +++ b/inputs/elasticsearch/fixtures/datastream/7.15.0.json @@ -0,0 +1,24 @@ +{ + "_shards": { + "failed": 0, + "successful": 30, + "total": 30 + }, + "backing_indices": 7, + "data_stream_count": 2, + "data_streams": [ + { + "backing_indices": 5, + "data_stream": "foo", + "maximum_timestamp": 1656079894000, + "store_size_bytes": 429205396 + }, + { + "backing_indices": 2, + "data_stream": "bar", + "maximum_timestamp": 1656028796000, + "store_size_bytes": 673822720 + } + ], + "total_store_size_bytes": 1103028116 +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/ilm_indices/6.6.0.json b/inputs/elasticsearch/fixtures/ilm_indices/6.6.0.json new file mode 100644 index 00000000..ff647b71 --- /dev/null +++ b/inputs/elasticsearch/fixtures/ilm_indices/6.6.0.json @@ -0,0 +1,20 @@ +{ + "indices": { + "facebook": { + "action": "complete", + "action_time_millis": 1660799138651, + "index": "facebook", + "lifecycle_date_millis": 1660799138565, + "managed": true, + "phase": "new", + "phase_time_millis": 1660799138651, + "policy": "my_policy", + "step": "complete", + "step_time_millis": 1660799138651 + }, + "twitter": { + "index": "twitter", + "managed": false + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/ilm_status/6.6.0.json b/inputs/elasticsearch/fixtures/ilm_status/6.6.0.json new file mode 100644 index 00000000..b07355e5 --- /dev/null +++ b/inputs/elasticsearch/fixtures/ilm_status/6.6.0.json @@ -0,0 +1,3 @@ +{ + "operation_mode": "RUNNING" +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/indices_mappings/7.8.0.json b/inputs/elasticsearch/fixtures/indices_mappings/7.8.0.json new file mode 100644 index 00000000..0356bc57 --- /dev/null +++ b/inputs/elasticsearch/fixtures/indices_mappings/7.8.0.json @@ -0,0 +1,43 @@ +{ + "facebook": { + "mappings": { + "properties": { + "contact": { + "properties": { + "email": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "phone": { + "type": "text" + } + } + }, + "name": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + } + } + } + }, + "twitter": { + "mappings": { + "properties": { + "email": { + "type": "keyword" + }, + "phone": { + "type": "keyword" + } + } + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/indices_mappings/counts.json b/inputs/elasticsearch/fixtures/indices_mappings/counts.json new file mode 100644 index 00000000..4cb5389e --- /dev/null +++ b/inputs/elasticsearch/fixtures/indices_mappings/counts.json @@ -0,0 +1,183 @@ +{ + "test-data-2023.01.20": { + "mappings": { + "properties": { + "data": { + "properties": { + "field1": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field10": { + "type": "long" + }, + "field2": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field3": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field4": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field5": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field6": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field7": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field8": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field9": { + "type": "long" + } + }, + "type": "object" + }, + "data2": { + "properties": { + "field1": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field2": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field3": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field4": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field5": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "nested_field6": { + "properties": { + "field1": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field2": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field3": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field4": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "field5": { + "type": "long" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/5.4.2.json b/inputs/elasticsearch/fixtures/nodestats/5.4.2.json new file mode 100644 index 00000000..d1e1baf2 --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/5.4.2.json @@ -0,0 +1,458 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "oDXKc-AOSEG9KfDb-qc7Vg": { + "breakers": { + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.1gb", + "limit_size_in_bytes": 1246652006, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.9gb", + "limit_size_in_bytes": 2077753344, + "overhead": 1, + "tripped": 0 + }, + "parent": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.3gb", + "limit_size_in_bytes": 1454427340, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.1gb", + "limit_size_in_bytes": 1246652006, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": { + "cluster_state_queue": { + "committed": 0, + "pending": 0, + "total": 0 + } + }, + "fs": { + "data": [ + { + "available_in_bytes": 77533327360, + "free_in_bytes": 77533327360, + "mount": "/usr/share/elasticsearch/data (/dev/mapper/vg0-root)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "spins": "true", + "total_in_bytes": 476630163456, + "type": "xfs" + } + ], + "io_stats": { + "devices": [ + { + "device_name": "dm-2", + "operations": 5916, + "read_kilobytes": 71588, + "read_operations": 4854, + "write_kilobytes": 8340, + "write_operations": 1062 + } + ], + "total": { + "operations": 5916, + "read_kilobytes": 71588, + "read_operations": 4854, + "write_kilobytes": 8340, + "write_operations": 1062 + } + }, + "timestamp": 1622148124418, + "total": { + "available_in_bytes": 77533327360, + "free_in_bytes": 77533327360, + "spins": "true", + "total_in_bytes": 476630163456 + } + }, + "host": "127.0.0.1", + "http": { + "current_open": 1, + "total_opened": 16 + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 47, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 629145600, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "listeners": 0, + "total": 5, + "total_time_in_millis": 98 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 460, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 12940, + "norms_memory_in_bytes": 960, + "points_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 1560, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 9960, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 5337, + "throttle_time_in_millis": 0 + }, + "translog": { + "operations": 5, + "size_in_bytes": 1743 + }, + "warmer": { + "current": 0, + "total": 35, + "total_time_in_millis": 4 + } + }, + "ingest": { + "pipelines": {}, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "127.0.0.1:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 47, + "total_capacity_in_bytes": 252727740, + "used_in_bytes": 252727741 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 15007, + "used_in_bytes": 15007 + } + }, + "classes": { + "current_loaded_count": 10411, + "total_loaded_count": 10411, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 1, + "collection_time_in_millis": 120 + }, + "young": { + "collection_count": 1, + "collection_time_in_millis": 24 + } + } + }, + "mem": { + "heap_committed_in_bytes": 2077753344, + "heap_max_in_bytes": 2077753344, + "heap_used_in_bytes": 569163128, + "heap_used_percent": 27, + "non_heap_committed_in_bytes": 73859072, + "non_heap_used_in_bytes": 68846616, + "pools": { + "old": { + "max_in_bytes": 1449590784, + "peak_max_in_bytes": 1449590784, + "peak_used_in_bytes": 0, + "used_in_bytes": 0 + }, + "survivor": { + "max_in_bytes": 69730304, + "peak_max_in_bytes": 69730304, + "peak_used_in_bytes": 50615520, + "used_in_bytes": 50615520 + }, + "young": { + "max_in_bytes": 558432256, + "peak_max_in_bytes": 558432256, + "peak_used_in_bytes": 558432256, + "used_in_bytes": 518547608 + } + } + }, + "threads": { + "count": 61, + "peak_count": 61 + }, + "timestamp": 1622148124416, + "uptime_in_millis": 13951 + }, + "name": "oDXKc-A", + "os": { + "cpu": { + "load_average": { + "15m": 98.03, + "1m": 62.46, + "5m": 160.53 + }, + "percent": 29 + }, + "mem": { + "free_in_bytes": 7032000512, + "free_percent": 21, + "total_in_bytes": 33623236608, + "used_in_bytes": 26591236096, + "used_percent": 79 + }, + "swap": { + "free_in_bytes": 85319680, + "total_in_bytes": 2147479552, + "used_in_bytes": 2062159872 + }, + "timestamp": 1622148124416 + }, + "process": { + "cpu": { + "percent": 9, + "total_in_millis": 13610 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 8294236160 + }, + "open_file_descriptors": 305, + "timestamp": 1622148124416 + }, + "roles": [ + "master", + "data", + "ingest" + ], + "script": { + "cache_evictions": 0, + "compilations": 0 + }, + "thread_pool": { + "bulk": { + "active": 0, + "completed": 5, + "largest": 5, + "queue": 0, + "rejected": 0, + "threads": 5 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 38, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "index": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 5, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "management": { + "active": 1, + "completed": 2, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "refresh": { + "active": 0, + "completed": 31, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 5, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + } + }, + "timestamp": 1622148124411, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "127.0.0.1:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/5.6.16.json b/inputs/elasticsearch/fixtures/nodestats/5.6.16.json new file mode 100644 index 00000000..c5cd3f39 --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/5.6.16.json @@ -0,0 +1,458 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "bVrN1HxvQLy795ZLNg2XNw": { + "breakers": { + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.1gb", + "limit_size_in_bytes": 1246652006, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.9gb", + "limit_size_in_bytes": 2077753344, + "overhead": 1, + "tripped": 0 + }, + "parent": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.3gb", + "limit_size_in_bytes": 1454427340, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1.1gb", + "limit_size_in_bytes": 1246652006, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": { + "cluster_state_queue": { + "committed": 0, + "pending": 0, + "total": 0 + } + }, + "fs": { + "data": [ + { + "available_in_bytes": 77533405184, + "free_in_bytes": 77533405184, + "mount": "/usr/share/elasticsearch/data (/dev/mapper/vg0-root)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "spins": "true", + "total_in_bytes": 476630163456, + "type": "xfs" + } + ], + "io_stats": { + "devices": [ + { + "device_name": "dm-2", + "operations": 2517, + "read_kilobytes": 12916, + "read_operations": 706, + "write_kilobytes": 17760, + "write_operations": 1811 + } + ], + "total": { + "operations": 2517, + "read_kilobytes": 12916, + "read_operations": 706, + "write_kilobytes": 17760, + "write_operations": 1811 + } + }, + "timestamp": 1622148141244, + "total": { + "available_in_bytes": 77533405184, + "free_in_bytes": 77533405184, + "spins": "true", + "total_in_bytes": 476630163456 + } + }, + "host": "127.0.0.1", + "http": { + "current_open": 1, + "total_opened": 16 + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 39, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 629145600, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "listeners": 0, + "total": 65, + "total_time_in_millis": 86 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 460, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 12940, + "norms_memory_in_bytes": 960, + "points_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 1560, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 9960, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 5721, + "throttle_time_in_millis": 0 + }, + "translog": { + "operations": 5, + "size_in_bytes": 1743 + }, + "warmer": { + "current": 0, + "total": 35, + "total_time_in_millis": 1 + } + }, + "ingest": { + "pipelines": {}, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "127.0.0.1:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 47, + "total_capacity_in_bytes": 252727868, + "used_in_bytes": 252727869 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 15007, + "used_in_bytes": 15007 + } + }, + "classes": { + "current_loaded_count": 10502, + "total_loaded_count": 10502, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 1, + "collection_time_in_millis": 109 + }, + "young": { + "collection_count": 2, + "collection_time_in_millis": 143 + } + } + }, + "mem": { + "heap_committed_in_bytes": 2077753344, + "heap_max_in_bytes": 2077753344, + "heap_used_in_bytes": 333706928, + "heap_used_percent": 16, + "non_heap_committed_in_bytes": 75362304, + "non_heap_used_in_bytes": 70212664, + "pools": { + "old": { + "max_in_bytes": 1449590784, + "peak_max_in_bytes": 1449590784, + "peak_used_in_bytes": 210051288, + "used_in_bytes": 210051288 + }, + "survivor": { + "max_in_bytes": 69730304, + "peak_max_in_bytes": 69730304, + "peak_used_in_bytes": 69730304, + "used_in_bytes": 69730304 + }, + "young": { + "max_in_bytes": 558432256, + "peak_max_in_bytes": 558432256, + "peak_used_in_bytes": 558432256, + "used_in_bytes": 53925336 + } + } + }, + "threads": { + "count": 60, + "peak_count": 60 + }, + "timestamp": 1622148141243, + "uptime_in_millis": 14845 + }, + "name": "bVrN1Hx", + "os": { + "cpu": { + "load_average": { + "15m": 96.01, + "1m": 45.68, + "5m": 150.34 + }, + "percent": 23 + }, + "mem": { + "free_in_bytes": 7009173504, + "free_percent": 21, + "total_in_bytes": 33623236608, + "used_in_bytes": 26614063104, + "used_percent": 79 + }, + "swap": { + "free_in_bytes": 86630400, + "total_in_bytes": 2147479552, + "used_in_bytes": 2060849152 + }, + "timestamp": 1622148141242 + }, + "process": { + "cpu": { + "percent": 8, + "total_in_millis": 14510 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 8293711872 + }, + "open_file_descriptors": 308, + "timestamp": 1622148141242 + }, + "roles": [ + "master", + "data", + "ingest" + ], + "script": { + "cache_evictions": 0, + "compilations": 0 + }, + "thread_pool": { + "bulk": { + "active": 0, + "completed": 5, + "largest": 5, + "queue": 0, + "rejected": 0, + "threads": 5 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 38, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "index": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 5, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "management": { + "active": 1, + "completed": 2, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "refresh": { + "active": 0, + "completed": 31, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + } + }, + "timestamp": 1622148141237, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "127.0.0.1:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/6.5.4.json b/inputs/elasticsearch/fixtures/nodestats/6.5.4.json new file mode 100644 index 00000000..ee33af32 --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/6.5.4.json @@ -0,0 +1,591 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "Do51G27MTS-Z39E-5FnQgA": { + "adaptive_selection": {}, + "attributes": { + "ml.enabled": "true", + "ml.machine_memory": "33623236608", + "ml.max_open_jobs": "20", + "xpack.installed": "true" + }, + "breakers": { + "accounting": { + "estimated_size": "8.2kb", + "estimated_size_in_bytes": 8490, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "593.9mb", + "limit_size_in_bytes": 622775500, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 1, + "tripped": 0 + }, + "parent": { + "estimated_size": "8.2kb", + "estimated_size_in_bytes": 8490, + "limit_size": "692.9mb", + "limit_size_in_bytes": 726571417, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "593.9mb", + "limit_size_in_bytes": 622775500, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": {}, + "fs": { + "data": [ + { + "available_in_bytes": 77532463104, + "free_in_bytes": 77532463104, + "mount": "/ (overlay)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes": 476630163456, + "type": "overlay" + } + ], + "io_stats": {}, + "timestamp": 1622148161618, + "total": { + "available_in_bytes": 77532463104, + "free_in_bytes": 77532463104, + "total_in_bytes": 476630163456 + } + }, + "host": "172.17.0.2", + "http": { + "current_open": 1, + "total_opened": 16 + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "periodic": 0, + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 39, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 629145600, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "listeners": 0, + "total": 65, + "total_time_in_millis": 101 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 340, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 8490, + "norms_memory_in_bytes": 640, + "points_memory_in_bytes": 5, + "stored_fields_memory_in_bytes": 1560, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 5945, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 6298 + }, + "translog": { + "earliest_last_modified_age": 0, + "operations": 5, + "size_in_bytes": 3753, + "uncommitted_operations": 5, + "uncommitted_size_in_bytes": 3753 + }, + "warmer": { + "current": 0, + "total": 35, + "total_time_in_millis": 0 + } + }, + "ingest": { + "pipelines": { + "xpack_monitoring_2": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [ + { + "script": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + { + "rename": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + { + "set": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + { + "gsub": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + } + ], + "time_in_millis": 0 + }, + "xpack_monitoring_6": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [], + "time_in_millis": 0 + } + }, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "172.17.0.2:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 41, + "total_capacity_in_bytes": 168846950, + "used_in_bytes": 168846951 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 15169, + "used_in_bytes": 15169 + } + }, + "classes": { + "current_loaded_count": 15646, + "total_loaded_count": 15646, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 2, + "collection_time_in_millis": 118 + }, + "young": { + "collection_count": 6, + "collection_time_in_millis": 175 + } + } + }, + "mem": { + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 393589472, + "heap_used_percent": 37, + "non_heap_committed_in_bytes": 119910400, + "non_heap_used_in_bytes": 110886752, + "pools": { + "old": { + "max_in_bytes": 715849728, + "peak_max_in_bytes": 715849728, + "peak_used_in_bytes": 137330480, + "used_in_bytes": 137330480 + }, + "survivor": { + "max_in_bytes": 35782656, + "peak_max_in_bytes": 35782656, + "peak_used_in_bytes": 35782656, + "used_in_bytes": 35782656 + }, + "young": { + "max_in_bytes": 286326784, + "peak_max_in_bytes": 286326784, + "peak_used_in_bytes": 286326784, + "used_in_bytes": 220476336 + } + } + }, + "threads": { + "count": 49, + "peak_count": 49 + }, + "timestamp": 1622148161615, + "uptime_in_millis": 18361 + }, + "name": "Do51G27", + "os": { + "cgroup": { + "cpu": { + "cfs_period_micros": 100000, + "cfs_quota_micros": -1, + "control_group": "/", + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + }, + "cpuacct": { + "control_group": "/", + "usage_nanos": 37809554078 + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "9223372036854771712", + "usage_in_bytes": "1640902656" + } + }, + "cpu": { + "load_average": { + "15m": 94.1, + "1m": 34.46, + "5m": 141.02 + }, + "percent": 34 + }, + "mem": { + "free_in_bytes": 7655825408, + "free_percent": 23, + "total_in_bytes": 33623236608, + "used_in_bytes": 25967411200, + "used_percent": 77 + }, + "swap": { + "free_in_bytes": 108126208, + "total_in_bytes": 2147479552, + "used_in_bytes": 2039353344 + }, + "timestamp": 1622148161614 + }, + "process": { + "cpu": { + "percent": 19, + "total_in_millis": 37420 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 7215165440 + }, + "open_file_descriptors": 402, + "timestamp": 1622148161614 + }, + "roles": [ + "master", + "data", + "ingest" + ], + "script": { + "cache_evictions": 0, + "compilations": 4 + }, + "thread_pool": { + "analyze": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ccr": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 54, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "index": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 5, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "management": { + "active": 1, + "completed": 4, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "ml_autodetect": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_datafeed": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_utility": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "refresh": { + "active": 0, + "completed": 32, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "rollup_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search_throttled": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "security-token-key": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "watcher": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "write": { + "active": 0, + "completed": 5, + "largest": 5, + "queue": 0, + "rejected": 0, + "threads": 5 + } + }, + "timestamp": 1622148161602, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "172.17.0.2:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/6.8.8.json b/inputs/elasticsearch/fixtures/nodestats/6.8.8.json new file mode 100644 index 00000000..2dbdf5ca --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/6.8.8.json @@ -0,0 +1,596 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "9_P7yuiySjG7OAN6NRbBRA": { + "adaptive_selection": {}, + "attributes": { + "ml.enabled": "true", + "ml.machine_memory": "33623236608", + "ml.max_open_jobs": "20", + "xpack.installed": "true" + }, + "breakers": { + "accounting": { + "estimated_size": "8.2kb", + "estimated_size_in_bytes": 8490, + "limit_size": "1gb", + "limit_size_in_bytes": 1073741824, + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "614.3mb", + "limit_size_in_bytes": 644245094, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "1gb", + "limit_size_in_bytes": 1073741824, + "overhead": 1, + "tripped": 0 + }, + "parent": { + "estimated_size": "8.2kb", + "estimated_size_in_bytes": 8490, + "limit_size": "716.7mb", + "limit_size_in_bytes": 751619276, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "614.3mb", + "limit_size_in_bytes": 644245094, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": {}, + "fs": { + "data": [ + { + "available_in_bytes": 77532815360, + "free_in_bytes": 77532815360, + "mount": "/ (overlay)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes": 476630163456, + "type": "overlay" + } + ], + "io_stats": {}, + "timestamp": 1622148180551, + "total": { + "available_in_bytes": 77532815360, + "free_in_bytes": 77532815360, + "total_in_bytes": 476630163456 + } + }, + "host": "172.17.0.2", + "http": { + "current_open": 1, + "total_opened": 16 + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "periodic": 0, + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 38, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 629145600, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "listeners": 0, + "total": 65, + "total_time_in_millis": 103 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 340, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 8490, + "norms_memory_in_bytes": 640, + "points_memory_in_bytes": 5, + "stored_fields_memory_in_bytes": 1560, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 5945, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 7261 + }, + "translog": { + "earliest_last_modified_age": 0, + "operations": 5, + "size_in_bytes": 3753, + "uncommitted_operations": 5, + "uncommitted_size_in_bytes": 3753 + }, + "warmer": { + "current": 0, + "total": 35, + "total_time_in_millis": 0 + } + }, + "ingest": { + "pipelines": { + "xpack_monitoring_2": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [ + { + "script": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + { + "rename": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + { + "set": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + { + "gsub": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + } + ], + "time_in_millis": 0 + }, + "xpack_monitoring_6": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [], + "time_in_millis": 0 + } + }, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "172.17.0.2:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 42, + "total_capacity_in_bytes": 168849055, + "used_in_bytes": 168849056 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 15179, + "used_in_bytes": 15179 + }, + "mapped - 'non-volatile memory'": { + "count": 0, + "total_capacity_in_bytes": 0, + "used_in_bytes": 0 + } + }, + "classes": { + "current_loaded_count": 16278, + "total_loaded_count": 16278, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + }, + "young": { + "collection_count": 5, + "collection_time_in_millis": 80 + } + } + }, + "mem": { + "heap_committed_in_bytes": 1073741824, + "heap_max_in_bytes": 1073741824, + "heap_used_in_bytes": 660054016, + "heap_used_percent": 61, + "non_heap_committed_in_bytes": 117964800, + "non_heap_used_in_bytes": 108594112, + "pools": { + "old": { + "max_in_bytes": 1073741824, + "peak_max_in_bytes": 1073741824, + "peak_used_in_bytes": 255827968, + "used_in_bytes": 255827968 + }, + "survivor": { + "max_in_bytes": 0, + "peak_max_in_bytes": 0, + "peak_used_in_bytes": 47668160, + "used_in_bytes": 11010048 + }, + "young": { + "max_in_bytes": 0, + "peak_max_in_bytes": 0, + "peak_used_in_bytes": 559417344, + "used_in_bytes": 393216000 + } + } + }, + "threads": { + "count": 54, + "peak_count": 54 + }, + "timestamp": 1622148180548, + "uptime_in_millis": 16456 + }, + "name": "9_P7yui", + "os": { + "cgroup": { + "cpu": { + "cfs_period_micros": 100000, + "cfs_quota_micros": -1, + "control_group": "/", + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + }, + "cpuacct": { + "control_group": "/", + "usage_nanos": 33206615382 + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "9223372036854771712", + "usage_in_bytes": "1634058240" + } + }, + "cpu": { + "load_average": { + "15m": 92.65, + "1m": 27.55, + "5m": 134.28 + }, + "percent": 30 + }, + "mem": { + "free_in_bytes": 7651008512, + "free_percent": 23, + "total_in_bytes": 33623236608, + "used_in_bytes": 25972228096, + "used_percent": 77 + }, + "swap": { + "free_in_bytes": 109174784, + "total_in_bytes": 2147479552, + "used_in_bytes": 2038304768 + }, + "timestamp": 1622148180547 + }, + "process": { + "cpu": { + "percent": 17, + "total_in_millis": 32810 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 7269961728 + }, + "open_file_descriptors": 355, + "timestamp": 1622148180547 + }, + "roles": [ + "master", + "data", + "ingest" + ], + "script": { + "cache_evictions": 0, + "compilations": 1 + }, + "thread_pool": { + "analyze": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ccr": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 87, + "largest": 5, + "queue": 0, + "rejected": 0, + "threads": 5 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "index": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 5, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "management": { + "active": 1, + "completed": 5, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "ml_autodetect": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_datafeed": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_utility": { + "active": 0, + "completed": 1, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "refresh": { + "active": 0, + "completed": 32, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "rollup_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search_throttled": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "security-token-key": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "watcher": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "write": { + "active": 0, + "completed": 5, + "largest": 5, + "queue": 0, + "rejected": 0, + "threads": 5 + } + }, + "timestamp": 1622148180536, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "172.17.0.2:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/7.13.1.json b/inputs/elasticsearch/fixtures/nodestats/7.13.1.json new file mode 100644 index 00000000..ec2d723d --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/7.13.1.json @@ -0,0 +1,1094 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "byoDEtBRSRGZyMKaIpmhCQ": { + "adaptive_selection": {}, + "attributes": { + "ml.machine_memory": "33623232512", + "ml.max_jvm_size": "788529152", + "ml.max_open_jobs": "512", + "transform.node": "true", + "xpack.installed": "true" + }, + "breakers": { + "accounting": { + "estimated_size": "9.1kb", + "estimated_size_in_bytes": 9380, + "limit_size": "752mb", + "limit_size_in_bytes": 788529152, + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "300.7mb", + "limit_size_in_bytes": 315411660, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "752mb", + "limit_size_in_bytes": 788529152, + "overhead": 2, + "tripped": 0 + }, + "model_inference": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "376mb", + "limit_size_in_bytes": 394264576, + "overhead": 1, + "tripped": 0 + }, + "parent": { + "estimated_size": "148.9mb", + "estimated_size_in_bytes": 156194432, + "limit_size": "714.3mb", + "limit_size_in_bytes": 749102694, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "451.1mb", + "limit_size_in_bytes": 473117491, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": { + "cluster_state_queue": { + "committed": 0, + "pending": 0, + "total": 0 + }, + "published_cluster_states": { + "compatible_diffs": 49, + "full_states": 2, + "incompatible_diffs": 0 + } + }, + "fs": { + "data": [ + { + "available_in_bytes": 63425642496, + "free_in_bytes": 63425642496, + "mount": "/ (overlay)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes": 476630163456, + "type": "overlay" + } + ], + "io_stats": {}, + "timestamp": 1622732353202, + "total": { + "available_in_bytes": 63425642496, + "free_in_bytes": 63425642496, + "total_in_bytes": 476630163456 + } + }, + "host": "172.17.0.2", + "http": { + "clients": [ + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732350948, + "id": 789026726, + "last_request_time_millis": 1622732349747, + "last_uri": "/foo_2/type1/1?refresh=wait_for", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732349747, + "remote_address": "172.17.0.1:49438", + "request_count": 1, + "request_size_bytes": 39 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732353149, + "id": 150912494, + "last_request_time_millis": 1622732353149, + "last_uri": "/_all/_stats", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732353149, + "remote_address": "172.17.0.1:49458", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732351948, + "id": 1204144009, + "last_request_time_millis": 1622732350948, + "last_uri": "/foo_2/type1/2?refresh=wait_for", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732350948, + "remote_address": "172.17.0.1:49442", + "request_count": 1, + "request_size_bytes": 39 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732345338, + "id": 832608635, + "last_request_time_millis": 1622732343337, + "last_uri": "/twitter", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732343337, + "remote_address": "172.17.0.1:49402", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732353149, + "id": 1787075320, + "last_request_time_millis": 1622732353149, + "last_uri": "/_all/_settings", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732353149, + "remote_address": "172.17.0.1:49454", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732347139, + "id": 1263458828, + "last_request_time_millis": 1622732346538, + "last_uri": "/viber", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732346538, + "remote_address": "172.17.0.1:49416", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732352949, + "id": 186367096, + "last_request_time_millis": 1622732352949, + "last_uri": "/_cluster/health", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732352949, + "remote_address": "172.17.0.1:49450", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732343337, + "id": 1263584333, + "last_request_time_millis": 1622732343337, + "last_uri": "/", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732343337, + "remote_address": "172.17.0.1:49398", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732347339, + "id": 1290037325, + "last_request_time_millis": 1622732347139, + "last_uri": "/instagram/_settings", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732347139, + "remote_address": "172.17.0.1:49420", + "request_count": 1, + "request_size_bytes": 124 + }, + { + "agent": "curl/7.68.0", + "id": 170704265, + "last_request_time_millis": 1622732353149, + "last_uri": "/_nodes/stats", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732353149, + "remote_address": "172.17.0.1:49462", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732348747, + "id": 1002191851, + "last_request_time_millis": 1622732347539, + "last_uri": "/foo_1/type1/1?refresh=wait_for", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732347539, + "remote_address": "172.17.0.1:49428", + "request_count": 1, + "request_size_bytes": 33 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732347539, + "id": 119489056, + "last_request_time_millis": 1622732347339, + "last_uri": "/twitter/_settings", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732347339, + "remote_address": "172.17.0.1:49424", + "request_count": 1, + "request_size_bytes": 124 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732352949, + "id": 562495253, + "last_request_time_millis": 1622732351948, + "last_uri": "/foo_2/type1/3?refresh=wait_for", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732351948, + "remote_address": "172.17.0.1:49446", + "request_count": 1, + "request_size_bytes": 39 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732346538, + "id": 1052374837, + "last_request_time_millis": 1622732345938, + "last_uri": "/instagram", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732345938, + "remote_address": "172.17.0.1:49412", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732345938, + "id": 1677987869, + "last_request_time_millis": 1622732345338, + "last_uri": "/facebook", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732345338, + "remote_address": "172.17.0.1:49408", + "request_count": 1, + "request_size_bytes": 0 + }, + { + "agent": "curl/7.68.0", + "closed_time_millis": 1622732349747, + "id": 977313322, + "last_request_time_millis": 1622732348747, + "last_uri": "/foo_1/type1/2?refresh=wait_for", + "local_address": "172.17.0.2:9200", + "opened_time_millis": 1622732348747, + "remote_address": "172.17.0.1:49432", + "request_count": 1, + "request_size_bytes": 33 + } + ], + "current_open": 1, + "total_opened": 16 + }, + "indexing_pressure": { + "memory": { + "current": { + "all_in_bytes": 0, + "combined_coordinating_and_primary_in_bytes": 0, + "coordinating_in_bytes": 0, + "primary_in_bytes": 0, + "replica_in_bytes": 0 + }, + "limit_in_bytes": 78852915, + "total": { + "all_in_bytes": 1423, + "combined_coordinating_and_primary_in_bytes": 1423, + "coordinating_in_bytes": 1423, + "coordinating_rejections": 0, + "primary_in_bytes": 1463, + "primary_rejections": 0, + "replica_in_bytes": 0, + "replica_rejections": 0 + } + } + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "periodic": 0, + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 14, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 125829120, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "external_total": 17, + "external_total_time_in_millis": 150, + "listeners": 0, + "total": 17, + "total_time_in_millis": 148 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 380, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 9380, + "norms_memory_in_bytes": 640, + "points_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 2440, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 5920, + "version_map_memory_in_bytes": 0 + }, + "store": { + "reserved_in_bytes": 0, + "size_in_bytes": 22296, + "total_data_set_size_in_bytes": 22296 + }, + "translog": { + "earliest_last_modified_age": 1118, + "operations": 5, + "size_in_bytes": 773, + "uncommitted_operations": 5, + "uncommitted_size_in_bytes": 773 + }, + "warmer": { + "current": 0, + "total": 11, + "total_time_in_millis": 0 + } + }, + "ingest": { + "pipelines": { + "xpack_monitoring_6": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [ + { + "script": { + "stats": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + }, + "type": "script" + } + }, + { + "gsub": { + "stats": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + }, + "type": "gsub" + } + } + ], + "time_in_millis": 0 + }, + "xpack_monitoring_7": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [], + "time_in_millis": 0 + } + }, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "172.17.0.2:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 33, + "total_capacity_in_bytes": 8811045, + "used_in_bytes": 8811046 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 16868, + "used_in_bytes": 16868 + }, + "mapped - 'non-volatile memory'": { + "count": 0, + "total_capacity_in_bytes": 0, + "used_in_bytes": 0 + } + }, + "classes": { + "current_loaded_count": 21909, + "total_loaded_count": 21909, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + }, + "young": { + "collection_count": 11, + "collection_time_in_millis": 113 + } + } + }, + "mem": { + "heap_committed_in_bytes": 788529152, + "heap_max_in_bytes": 788529152, + "heap_used_in_bytes": 156194432, + "heap_used_percent": 19, + "non_heap_committed_in_bytes": 142606336, + "non_heap_used_in_bytes": 139526472, + "pools": { + "old": { + "max_in_bytes": 788529152, + "peak_max_in_bytes": 788529152, + "peak_used_in_bytes": 71059968, + "used_in_bytes": 71059968 + }, + "survivor": { + "max_in_bytes": 0, + "peak_max_in_bytes": 0, + "peak_used_in_bytes": 54525952, + "used_in_bytes": 30608512 + }, + "young": { + "max_in_bytes": 0, + "peak_max_in_bytes": 0, + "peak_used_in_bytes": 465567744, + "used_in_bytes": 54525952 + } + } + }, + "threads": { + "count": 49, + "peak_count": 49 + }, + "timestamp": 1622732353200, + "uptime_in_millis": 21844 + }, + "name": "aaf5a8a0bceb", + "os": { + "cgroup": { + "cpu": { + "cfs_period_micros": 100000, + "cfs_quota_micros": -1, + "control_group": "/", + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + }, + "cpuacct": { + "control_group": "/", + "usage_nanos": 52445263941 + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "9223372036854771712", + "usage_in_bytes": "1110376448" + } + }, + "cpu": { + "load_average": { + "15m": 70.55, + "1m": 2.74, + "5m": 44.07 + }, + "percent": 37 + }, + "mem": { + "free_in_bytes": 12400742400, + "free_percent": 37, + "total_in_bytes": 33623232512, + "used_in_bytes": 21222490112, + "used_percent": 63 + }, + "swap": { + "free_in_bytes": 928747520, + "total_in_bytes": 2147479552, + "used_in_bytes": 1218732032 + }, + "timestamp": 1622732353199 + }, + "process": { + "cpu": { + "percent": 19, + "total_in_millis": 50160 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 6819479552 + }, + "open_file_descriptors": 314, + "timestamp": 1622732353199 + }, + "roles": [ + "data", + "data_cold", + "data_content", + "data_frozen", + "data_hot", + "data_warm", + "ingest", + "master", + "ml", + "remote_cluster_client", + "transform" + ], + "script": { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 1 + }, + "script_cache": { + "contexts": [ + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "aggregation_selector" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "aggs" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "aggs_combine" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "aggs_init" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "aggs_map" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "aggs_reduce" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "analysis" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "boolean_field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "bucket_aggregation" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "date_field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "double_field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "filter" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "geo_point_field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 1, + "context": "ingest" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "ingest_template" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "interval" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "ip_field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "keyword_field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "long_field" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "moving-function" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "number_sort" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "painless_test" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "processor_conditional" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "score" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "script_heuristic" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "similarity" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "similarity_weight" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "string_sort" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "template" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "terms_set" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "update" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "watcher_condition" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "watcher_transform" + }, + { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 0, + "context": "xpack_template" + } + ], + "sum": { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 1 + } + }, + "thread_pool": { + "analyze": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ccr": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 406, + "largest": 7, + "queue": 0, + "rejected": 0, + "threads": 7 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "management": { + "active": 1, + "completed": 9, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "ml_datafeed": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_job_comms": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_utility": { + "active": 0, + "completed": 12, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "refresh": { + "active": 0, + "completed": 36, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "rollup_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search_throttled": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "searchable_snapshots_cache_fetch_async": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "searchable_snapshots_cache_prewarming": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "security-crypto": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "security-token-key": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "system_read": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "system_write": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "transform_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "watcher": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "write": { + "active": 0, + "completed": 9, + "largest": 8, + "queue": 0, + "rejected": 0, + "threads": 8 + } + }, + "timestamp": 1622732353195, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "total_outbound_connections": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "172.17.0.2:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/7.3.0.json b/inputs/elasticsearch/fixtures/nodestats/7.3.0.json new file mode 100644 index 00000000..4d46880c --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/7.3.0.json @@ -0,0 +1,588 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "EHaRN-xuSp65CRUUBluUXA": { + "adaptive_selection": {}, + "attributes": { + "ml.machine_memory": "33623236608", + "ml.max_open_jobs": "20", + "xpack.installed": "true" + }, + "breakers": { + "accounting": { + "estimated_size": "9kb", + "estimated_size_in_bytes": 9230, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "395.9mb", + "limit_size_in_bytes": 415183667, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 2, + "tripped": 0 + }, + "parent": { + "estimated_size": "349.9mb", + "estimated_size_in_bytes": 366927688, + "limit_size": "940.3mb", + "limit_size_in_bytes": 986061209, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "593.9mb", + "limit_size_in_bytes": 622775500, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": { + "cluster_state_queue": { + "committed": 0, + "pending": 0, + "total": 0 + }, + "published_cluster_states": { + "compatible_diffs": 29, + "full_states": 2, + "incompatible_diffs": 0 + } + }, + "fs": { + "data": [ + { + "available_in_bytes": 77533048832, + "free_in_bytes": 77533048832, + "mount": "/ (overlay)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes": 476630163456, + "type": "overlay" + } + ], + "io_stats": {}, + "timestamp": 1622148201145, + "total": { + "available_in_bytes": 77533048832, + "free_in_bytes": 77533048832, + "total_in_bytes": 476630163456 + } + }, + "host": "172.17.0.2", + "http": { + "current_open": 1, + "total_opened": 16 + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "periodic": 0, + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 36, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 125829120, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "external_total": 17, + "external_total_time_in_millis": 115, + "listeners": 0, + "total": 17, + "total_time_in_millis": 113 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 340, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 9230, + "norms_memory_in_bytes": 640, + "points_memory_in_bytes": 5, + "stored_fields_memory_in_bytes": 1560, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 6685, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 20684 + }, + "translog": { + "earliest_last_modified_age": 0, + "operations": 5, + "size_in_bytes": 1103, + "uncommitted_operations": 5, + "uncommitted_size_in_bytes": 1103 + }, + "warmer": { + "current": 0, + "total": 11, + "total_time_in_millis": 0 + } + }, + "ingest": { + "pipelines": { + "xpack_monitoring_6": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [ + { + "script": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + { + "gsub": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + } + ], + "time_in_millis": 0 + }, + "xpack_monitoring_7": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [], + "time_in_millis": 0 + } + }, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "172.17.0.2:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 37, + "total_capacity_in_bytes": 1123678, + "used_in_bytes": 1123679 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 15404, + "used_in_bytes": 15404 + } + }, + "classes": { + "current_loaded_count": 16811, + "total_loaded_count": 16811, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 2, + "collection_time_in_millis": 49 + }, + "young": { + "collection_count": 6, + "collection_time_in_millis": 145 + } + } + }, + "mem": { + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 366927688, + "heap_used_percent": 35, + "non_heap_committed_in_bytes": 117559296, + "non_heap_used_in_bytes": 107659096, + "pools": { + "old": { + "max_in_bytes": 715849728, + "peak_max_in_bytes": 715849728, + "peak_used_in_bytes": 64668616, + "used_in_bytes": 64668616 + }, + "survivor": { + "max_in_bytes": 35782656, + "peak_max_in_bytes": 35782656, + "peak_used_in_bytes": 35782656, + "used_in_bytes": 18019680 + }, + "young": { + "max_in_bytes": 286326784, + "peak_max_in_bytes": 286326784, + "peak_used_in_bytes": 286326784, + "used_in_bytes": 284239392 + } + } + }, + "threads": { + "count": 54, + "peak_count": 54 + }, + "timestamp": 1622148201143, + "uptime_in_millis": 18070 + }, + "name": "19ca5e6b4d14", + "os": { + "cgroup": { + "cpu": { + "cfs_period_micros": 100000, + "cfs_quota_micros": -1, + "control_group": "/", + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + }, + "cpuacct": { + "control_group": "/", + "usage_nanos": 35712884556 + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "9223372036854771712", + "usage_in_bytes": "1458454528" + } + }, + "cpu": { + "load_average": { + "15m": 90.29, + "1m": 19.39, + "5m": 123.8 + }, + "percent": 30 + }, + "mem": { + "free_in_bytes": 7762477056, + "free_percent": 23, + "total_in_bytes": 33623236608, + "used_in_bytes": 25860759552, + "used_percent": 77 + }, + "swap": { + "free_in_bytes": 109961216, + "total_in_bytes": 2147479552, + "used_in_bytes": 2037518336 + }, + "timestamp": 1622148201142 + }, + "process": { + "cpu": { + "percent": 16, + "total_in_millis": 34670 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 7067267072 + }, + "open_file_descriptors": 297, + "timestamp": 1622148201142 + }, + "roles": [ + "ingest", + "master", + "data" + ], + "script": { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 1 + }, + "thread_pool": { + "analyze": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ccr": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "data_frame_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 194, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 5, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "management": { + "active": 1, + "completed": 41, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "ml_datafeed": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_job_comms": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_utility": { + "active": 0, + "completed": 1, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "refresh": { + "active": 0, + "completed": 36, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "rollup_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search_throttled": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "security-token-key": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "watcher": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "write": { + "active": 0, + "completed": 9, + "largest": 8, + "queue": 0, + "rejected": 0, + "threads": 8 + } + }, + "timestamp": 1622148201138, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "172.17.0.2:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/7.6.1.json b/inputs/elasticsearch/fixtures/nodestats/7.6.1.json new file mode 100644 index 00000000..66dc9e60 --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/7.6.1.json @@ -0,0 +1,595 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "3gLc6y8KTXSplnEAPkyyEA": { + "adaptive_selection": {}, + "attributes": { + "ml.machine_memory": "33623277568", + "ml.max_open_jobs": "20", + "xpack.installed": "true" + }, + "breakers": { + "accounting": { + "estimated_size": "8.6kb", + "estimated_size_in_bytes": 8825, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "395.9mb", + "limit_size_in_bytes": 415183667, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 2, + "tripped": 0 + }, + "parent": { + "estimated_size": "202.5mb", + "estimated_size_in_bytes": 212353288, + "limit_size": "940.3mb", + "limit_size_in_bytes": 986061209, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "593.9mb", + "limit_size_in_bytes": 622775500, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": { + "cluster_state_queue": { + "committed": 0, + "pending": 0, + "total": 0 + }, + "published_cluster_states": { + "compatible_diffs": 33, + "full_states": 2, + "incompatible_diffs": 0 + } + }, + "fs": { + "data": [ + { + "available_in_bytes": 400386580480, + "free_in_bytes": 400386580480, + "mount": "/ (overlay)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes": 476630163456, + "type": "overlay" + } + ], + "io_stats": {}, + "timestamp": 1586315984565, + "total": { + "available_in_bytes": 400386580480, + "free_in_bytes": 400386580480, + "total_in_bytes": 476630163456 + } + }, + "host": "172.17.0.2", + "http": { + "current_open": 1, + "total_opened": 16 + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "periodic": 0, + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 16, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 125829120, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "external_total": 17, + "external_total_time_in_millis": 101, + "listeners": 0, + "total": 17, + "total_time_in_millis": 99 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 340, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 8825, + "norms_memory_in_bytes": 640, + "points_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 1560, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 6285, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 20484 + }, + "translog": { + "earliest_last_modified_age": 0, + "operations": 5, + "size_in_bytes": 1103, + "uncommitted_operations": 5, + "uncommitted_size_in_bytes": 1103 + }, + "warmer": { + "current": 0, + "total": 11, + "total_time_in_millis": 0 + } + }, + "ingest": { + "pipelines": { + "xpack_monitoring_6": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [ + { + "script": { + "stats": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + }, + "type": "script" + } + }, + { + "gsub": { + "stats": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + }, + "type": "gsub" + } + } + ], + "time_in_millis": 0 + }, + "xpack_monitoring_7": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [], + "time_in_millis": 0 + } + }, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "172.17.0.2:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 34, + "total_capacity_in_bytes": 16835180, + "used_in_bytes": 16835181 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 15244, + "used_in_bytes": 15244 + } + }, + "classes": { + "current_loaded_count": 18151, + "total_loaded_count": 18151, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 2, + "collection_time_in_millis": 61 + }, + "young": { + "collection_count": 8, + "collection_time_in_millis": 166 + } + } + }, + "mem": { + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 212353288, + "heap_used_percent": 20, + "non_heap_committed_in_bytes": 127705088, + "non_heap_used_in_bytes": 116814664, + "pools": { + "old": { + "max_in_bytes": 715849728, + "peak_max_in_bytes": 715849728, + "peak_used_in_bytes": 59105848, + "used_in_bytes": 59105848 + }, + "survivor": { + "max_in_bytes": 35782656, + "peak_max_in_bytes": 35782656, + "peak_used_in_bytes": 35782656, + "used_in_bytes": 20338592 + }, + "young": { + "max_in_bytes": 286326784, + "peak_max_in_bytes": 286326784, + "peak_used_in_bytes": 286326784, + "used_in_bytes": 132908848 + } + } + }, + "threads": { + "count": 61, + "peak_count": 61 + }, + "timestamp": 1586315984563, + "uptime_in_millis": 18063 + }, + "name": "75bb834b2aed", + "os": { + "cgroup": { + "cpu": { + "cfs_period_micros": 100000, + "cfs_quota_micros": -1, + "control_group": "/", + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + }, + "cpuacct": { + "control_group": "/", + "usage_nanos": 39379293527 + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "9223372036854771712", + "usage_in_bytes": "1409101824" + } + }, + "cpu": { + "load_average": { + "15m": 1.79, + "1m": 2.48, + "5m": 1.93 + }, + "percent": 29 + }, + "mem": { + "free_in_bytes": 4478267392, + "free_percent": 13, + "total_in_bytes": 33623277568, + "used_in_bytes": 29145010176, + "used_percent": 87 + }, + "swap": { + "free_in_bytes": 2147479552, + "total_in_bytes": 2147479552, + "used_in_bytes": 0 + }, + "timestamp": 1586315984562 + }, + "process": { + "cpu": { + "percent": 18, + "total_in_millis": 38760 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 7081488384 + }, + "open_file_descriptors": 313, + "timestamp": 1586315984562 + }, + "roles": [ + "ingest", + "master", + "data", + "ml" + ], + "script": { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 1 + }, + "thread_pool": { + "analyze": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ccr": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 215, + "largest": 5, + "queue": 0, + "rejected": 0, + "threads": 5 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 5, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "management": { + "active": 1, + "completed": 10, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "ml_datafeed": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_job_comms": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_utility": { + "active": 0, + "completed": 1, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "refresh": { + "active": 0, + "completed": 33, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "rollup_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search_throttled": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "security-token-key": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 2, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "transform_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "watcher": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "write": { + "active": 0, + "completed": 9, + "largest": 8, + "queue": 0, + "rejected": 0, + "threads": 8 + } + }, + "timestamp": 1586315984557, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "172.17.0.2:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/nodestats/7.6.2.json b/inputs/elasticsearch/fixtures/nodestats/7.6.2.json new file mode 100644 index 00000000..0c9e1a38 --- /dev/null +++ b/inputs/elasticsearch/fixtures/nodestats/7.6.2.json @@ -0,0 +1,595 @@ +{ + "_nodes": { + "failed": 0, + "successful": 1, + "total": 1 + }, + "cluster_name": "elasticsearch", + "nodes": { + "rL7RmAMrSGGg9L7msPEPoQ": { + "adaptive_selection": {}, + "attributes": { + "ml.machine_memory": "33623236608", + "ml.max_open_jobs": "20", + "xpack.installed": "true" + }, + "breakers": { + "accounting": { + "estimated_size": "8.6kb", + "estimated_size_in_bytes": 8825, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "395.9mb", + "limit_size_in_bytes": 415183667, + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "989.8mb", + "limit_size_in_bytes": 1037959168, + "overhead": 2, + "tripped": 0 + }, + "parent": { + "estimated_size": "219.6mb", + "estimated_size_in_bytes": 230311656, + "limit_size": "940.3mb", + "limit_size_in_bytes": 986061209, + "overhead": 1, + "tripped": 0 + }, + "request": { + "estimated_size": "0b", + "estimated_size_in_bytes": 0, + "limit_size": "593.9mb", + "limit_size_in_bytes": 622775500, + "overhead": 1, + "tripped": 0 + } + }, + "discovery": { + "cluster_state_queue": { + "committed": 0, + "pending": 0, + "total": 0 + }, + "published_cluster_states": { + "compatible_diffs": 33, + "full_states": 2, + "incompatible_diffs": 0 + } + }, + "fs": { + "data": [ + { + "available_in_bytes": 77533065216, + "free_in_bytes": 77533065216, + "mount": "/ (overlay)", + "path": "/usr/share/elasticsearch/data/nodes/0", + "total_in_bytes": 476630163456, + "type": "overlay" + } + ], + "io_stats": {}, + "timestamp": 1622148228316, + "total": { + "available_in_bytes": 77533065216, + "free_in_bytes": 77533065216, + "total_in_bytes": 476630163456 + } + }, + "host": "172.17.0.2", + "http": { + "current_open": 1, + "total_opened": 16 + }, + "indices": { + "completion": { + "size_in_bytes": 0 + }, + "docs": { + "count": 5, + "deleted": 0 + }, + "fielddata": { + "evictions": 0, + "memory_size_in_bytes": 0 + }, + "flush": { + "periodic": 0, + "total": 0, + "total_time_in_millis": 0 + }, + "get": { + "current": 0, + "exists_time_in_millis": 0, + "exists_total": 0, + "missing_time_in_millis": 0, + "missing_total": 0, + "time_in_millis": 0, + "total": 0 + }, + "indexing": { + "delete_current": 0, + "delete_time_in_millis": 0, + "delete_total": 0, + "index_current": 0, + "index_failed": 0, + "index_time_in_millis": 18, + "index_total": 5, + "is_throttled": false, + "noop_update_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_auto_throttle_in_bytes": 125829120, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "cache_count": 0, + "cache_size": 0, + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0, + "total_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "refresh": { + "external_total": 17, + "external_total_time_in_millis": 100, + "listeners": 0, + "total": 17, + "total_time_in_millis": 98 + }, + "request_cache": { + "evictions": 0, + "hit_count": 0, + "memory_size_in_bytes": 0, + "miss_count": 0 + }, + "search": { + "fetch_current": 0, + "fetch_time_in_millis": 0, + "fetch_total": 0, + "open_contexts": 0, + "query_current": 0, + "query_time_in_millis": 0, + "query_total": 0, + "scroll_current": 0, + "scroll_time_in_millis": 0, + "scroll_total": 0, + "suggest_current": 0, + "suggest_time_in_millis": 0, + "suggest_total": 0 + }, + "segments": { + "count": 5, + "doc_values_memory_in_bytes": 340, + "file_sizes": {}, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -1, + "memory_in_bytes": 8825, + "norms_memory_in_bytes": 640, + "points_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 1560, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 6285, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 20484 + }, + "translog": { + "earliest_last_modified_age": 0, + "operations": 5, + "size_in_bytes": 1103, + "uncommitted_operations": 5, + "uncommitted_size_in_bytes": 1103 + }, + "warmer": { + "current": 0, + "total": 11, + "total_time_in_millis": 0 + } + }, + "ingest": { + "pipelines": { + "xpack_monitoring_6": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [ + { + "script": { + "stats": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + }, + "type": "script" + } + }, + { + "gsub": { + "stats": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + }, + "type": "gsub" + } + } + ], + "time_in_millis": 0 + }, + "xpack_monitoring_7": { + "count": 0, + "current": 0, + "failed": 0, + "processors": [], + "time_in_millis": 0 + } + }, + "total": { + "count": 0, + "current": 0, + "failed": 0, + "time_in_millis": 0 + } + }, + "ip": "172.17.0.2:9300", + "jvm": { + "buffer_pools": { + "direct": { + "count": 34, + "total_capacity_in_bytes": 16836573, + "used_in_bytes": 16836574 + }, + "mapped": { + "count": 5, + "total_capacity_in_bytes": 15244, + "used_in_bytes": 15244 + } + }, + "classes": { + "current_loaded_count": 18153, + "total_loaded_count": 18153, + "total_unloaded_count": 0 + }, + "gc": { + "collectors": { + "old": { + "collection_count": 2, + "collection_time_in_millis": 91 + }, + "young": { + "collection_count": 8, + "collection_time_in_millis": 145 + } + } + }, + "mem": { + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 229595816, + "heap_used_percent": 22, + "non_heap_committed_in_bytes": 128307200, + "non_heap_used_in_bytes": 116974928, + "pools": { + "old": { + "max_in_bytes": 715849728, + "peak_max_in_bytes": 715849728, + "peak_used_in_bytes": 49636624, + "used_in_bytes": 49460928 + }, + "survivor": { + "max_in_bytes": 35782656, + "peak_max_in_bytes": 35782656, + "peak_used_in_bytes": 35782656, + "used_in_bytes": 31989216 + }, + "young": { + "max_in_bytes": 286326784, + "peak_max_in_bytes": 286326784, + "peak_used_in_bytes": 286326784, + "used_in_bytes": 148145672 + } + } + }, + "threads": { + "count": 61, + "peak_count": 61 + }, + "timestamp": 1622148228314, + "uptime_in_millis": 23368 + }, + "name": "2c63e39ac5ce", + "os": { + "cgroup": { + "cpu": { + "cfs_period_micros": 100000, + "cfs_quota_micros": -1, + "control_group": "/", + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + }, + "cpuacct": { + "control_group": "/", + "usage_nanos": 38830394077 + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "9223372036854771712", + "usage_in_bytes": "1500422144" + } + }, + "cpu": { + "load_average": { + "15m": 88.08, + "1m": 15.13, + "5m": 114.43 + }, + "percent": 28 + }, + "mem": { + "free_in_bytes": 7612055552, + "free_percent": 23, + "total_in_bytes": 33623236608, + "used_in_bytes": 26011181056, + "used_percent": 77 + }, + "swap": { + "free_in_bytes": 112582656, + "total_in_bytes": 2147479552, + "used_in_bytes": 2034896896 + }, + "timestamp": 1622148228314 + }, + "process": { + "cpu": { + "percent": 14, + "total_in_millis": 38140 + }, + "max_file_descriptors": 1048576, + "mem": { + "total_virtual_in_bytes": 7082541056 + }, + "open_file_descriptors": 313, + "timestamp": 1622148228314 + }, + "roles": [ + "ingest", + "master", + "data", + "ml" + ], + "script": { + "cache_evictions": 0, + "compilation_limit_triggered": 0, + "compilations": 1 + }, + "thread_pool": { + "analyze": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ccr": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_started": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "fetch_shard_store": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "flush": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "force_merge": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "generic": { + "active": 0, + "completed": 221, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "get": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "listener": { + "active": 0, + "completed": 5, + "largest": 4, + "queue": 0, + "rejected": 0, + "threads": 4 + }, + "management": { + "active": 1, + "completed": 10, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "ml_datafeed": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_job_comms": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "ml_utility": { + "active": 0, + "completed": 1, + "largest": 1, + "queue": 0, + "rejected": 0, + "threads": 1 + }, + "refresh": { + "active": 0, + "completed": 47, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "rollup_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "search_throttled": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "security-token-key": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "snapshot": { + "active": 0, + "completed": 2, + "largest": 2, + "queue": 0, + "rejected": 0, + "threads": 2 + }, + "transform_indexing": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "warmer": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "watcher": { + "active": 0, + "completed": 0, + "largest": 0, + "queue": 0, + "rejected": 0, + "threads": 0 + }, + "write": { + "active": 0, + "completed": 9, + "largest": 8, + "queue": 0, + "rejected": 0, + "threads": 8 + } + }, + "timestamp": 1622148228310, + "transport": { + "rx_count": 0, + "rx_size_in_bytes": 0, + "server_open": 0, + "tx_count": 0, + "tx_size_in_bytes": 0 + }, + "transport_address": "172.17.0.2:9300" + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/settings-5.4.2.json b/inputs/elasticsearch/fixtures/settings-5.4.2.json new file mode 100644 index 00000000..3f6ccae6 --- /dev/null +++ b/inputs/elasticsearch/fixtures/settings-5.4.2.json @@ -0,0 +1,661 @@ +{ + "defaults": { + "action": { + "auto_create_index": "true", + "destructive_requires_name": "false", + "master": { + "force_local": "false" + }, + "search": { + "shard_count": { + "limit": "9223372036854775807" + } + } + }, + "bootstrap": { + "ctrlhandler": "true", + "memory_lock": "false", + "seccomp": "true", + "system_call_filter": "true" + }, + "cache": { + "recycler": { + "page": { + "limit": { + "heap": "10%" + }, + "type": "CONCURRENT", + "weight": { + "bytes": "1.0", + "ints": "1.0", + "longs": "1.0", + "objects": "0.1" + } + } + } + }, + "client": { + "transport": { + "ignore_cluster_name": "false", + "nodes_sampler_interval": "5s", + "ping_timeout": "5s", + "sniff": "false" + }, + "type": "node" + }, + "cluster": { + "blocks": { + "read_only": "false" + }, + "indices": { + "close": { + "enable": "true" + } + }, + "info": { + "update": { + "interval": "30s", + "timeout": "15s" + } + }, + "name": "elasticsearch", + "nodes": { + "reconnect_interval": "10s" + }, + "routing": { + "allocation": { + "allow_rebalance": "indices_all_active", + "awareness": { + "attributes": "" + }, + "balance": { + "index": "0.55", + "shard": "0.45", + "threshold": "1.0" + }, + "cluster_concurrent_rebalance": "2", + "disk": { + "include_relocations": "true", + "reroute_interval": "60s", + "threshold_enabled": "true", + "watermark": { + "high": "90%", + "low": "85%" + } + }, + "enable": "ALL", + "node_concurrent_incoming_recoveries": "2", + "node_concurrent_outgoing_recoveries": "2", + "node_concurrent_recoveries": "2", + "node_initial_primaries_recoveries": "4", + "same_shard": { + "host": "false" + }, + "snapshot": { + "relocation_enabled": "false" + }, + "total_shards_per_node": "-1", + "type": "balanced" + }, + "rebalance": { + "enable": "ALL" + } + }, + "service": { + "slow_task_logging_threshold": "30s" + } + }, + "default": { + "path": { + "conf": "", + "logs": "" + } + }, + "discovery": { + "initial_state_timeout": "30s", + "type": "zen", + "zen": { + "commit_timeout": "30s", + "fd": { + "connect_on_network_disconnect": "false", + "ping_interval": "1s", + "ping_retries": "3", + "ping_timeout": "30s", + "register_connection_listener": "true" + }, + "hosts_provider": null, + "join_retry_attempts": "3", + "join_retry_delay": "100ms", + "join_timeout": "60000ms", + "master_election": { + "ignore_non_master_pings": "false", + "wait_for_joins_timeout": "30000ms" + }, + "max_pings_from_another_master": "3", + "minimum_master_nodes": "-1", + "no_master_block": "write", + "ping": { + "unicast": { + "concurrent_connects": "10", + "hosts": { + "resolve_timeout": "5s" + } + } + }, + "ping_timeout": "3s", + "publish_diff": { + "enable": "true" + }, + "publish_timeout": "30s", + "send_leave_request": "true" + } + }, + "gateway": { + "expected_data_nodes": "-1", + "expected_master_nodes": "-1", + "expected_nodes": "-1", + "initial_shards": "quorum", + "recover_after_data_nodes": "-1", + "recover_after_master_nodes": "0", + "recover_after_nodes": "-1", + "recover_after_time": "0ms" + }, + "http": { + "bind_host": [ + "0.0.0.0" + ], + "compression": "true", + "compression_level": "3", + "content_type": { + "required": "false" + }, + "cors": { + "allow-credentials": "false", + "allow-headers": "X-Requested-With,Content-Type,Content-Length", + "allow-methods": "OPTIONS,HEAD,GET,POST,PUT,DELETE", + "allow-origin": "", + "enabled": "false", + "max-age": "1728000" + }, + "detailed_errors": { + "enabled": "true" + }, + "enabled": "true", + "host": [ + "0.0.0.0" + ], + "max_chunk_size": "8kb", + "max_content_length": "100mb", + "max_header_size": "8kb", + "max_initial_line_length": "4kb", + "netty": { + "max_composite_buffer_components": "-1", + "max_cumulation_buffer_capacity": "-1b", + "receive_predictor_max": "64kb", + "receive_predictor_min": "64kb", + "receive_predictor_size": "64kb", + "worker_count": "8" + }, + "pipelining": "true", + "pipelining.max_events": "10000", + "port": "9200-9300", + "publish_host": [ + "0.0.0.0" + ], + "publish_port": "-1", + "reset_cookies": "false", + "tcp": { + "blocking_server": "false", + "keep_alive": "true", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + }, + "tcp_no_delay": "true", + "type": "", + "type.default": "netty4" + }, + "index": { + "codec": "default", + "store": { + "fs": { + "fs_lock": "native" + }, + "type": "" + } + }, + "indices": { + "analysis": { + "hunspell": { + "dictionary": { + "ignore_case": "false", + "lazy": "false" + } + } + }, + "breaker": { + "fielddata": { + "limit": "60%", + "overhead": "1.03", + "type": "memory" + }, + "request": { + "limit": "60%", + "overhead": "1.0", + "type": "memory" + }, + "total": { + "limit": "70%" + }, + "type": "hierarchy" + }, + "cache": { + "cleanup_interval": "1m" + }, + "fielddata": { + "cache": { + "size": "-1b" + } + }, + "mapping": { + "dynamic_timeout": "30s" + }, + "memory": { + "index_buffer_size": "10%", + "interval": "5s", + "max_index_buffer_size": "-1b", + "min_index_buffer_size": "48mb", + "shard_inactive_time": "5m" + }, + "queries": { + "cache": { + "all_segments": "false", + "count": "10000", + "size": "10%" + } + }, + "query": { + "bool": { + "max_clause_count": "1024" + }, + "query_string": { + "allowLeadingWildcard": "true", + "analyze_wildcard": "false" + } + }, + "recovery": { + "internal_action_long_timeout": "1800000ms", + "internal_action_timeout": "15m", + "max_bytes_per_sec": "40mb", + "recovery_activity_timeout": "1800000ms", + "retry_delay_network": "5s", + "retry_delay_state_sync": "500ms" + }, + "requests": { + "cache": { + "expire": "0ms", + "size": "1%" + } + }, + "store": { + "delete": { + "shard": { + "timeout": "30s" + } + }, + "throttle": { + "max_bytes_per_sec": "0b", + "type": "NONE" + } + }, + "ttl": { + "interval": "60s" + } + }, + "logger": { + "level": "INFO" + }, + "monitor": { + "fs": { + "refresh_interval": "1s" + }, + "jvm": { + "gc": { + "enabled": "true", + "overhead": { + "debug": "10", + "info": "25", + "warn": "50" + }, + "refresh_interval": "1s" + }, + "refresh_interval": "1s" + }, + "os": { + "refresh_interval": "1s" + }, + "process": { + "refresh_interval": "1s" + } + }, + "network": { + "bind_host": [ + "_local_" + ], + "breaker": { + "inflight_requests": { + "limit": "100%", + "overhead": "1.0" + } + }, + "host": [ + "_local_" + ], + "publish_host": [ + "_local_" + ], + "server": "true", + "tcp": { + "blocking": "false", + "blocking_client": "false", + "blocking_server": "false", + "connect_timeout": "30s", + "keep_alive": "true", + "no_delay": "true", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + } + }, + "node": { + "add_lock_id_to_custom_path": "true", + "data": "true", + "enable_lucene_segment_infos_trace": "false", + "id": { + "seed": "0" + }, + "ingest": "true", + "local_storage": "true", + "master": "true", + "max_local_storage_nodes": "1", + "name": "DK8-2Lc", + "portsfile": "false" + }, + "path": { + "conf": "", + "home": "/usr/share/elasticsearch", + "logs": "/usr/share/elasticsearch/logs", + "scripts": "", + "shared_data": "" + }, + "pidfile": "", + "processors": "4", + "repositories": { + "fs": { + "chunk_size": "-1b", + "compress": "false", + "location": "" + }, + "url": { + "supported_protocols": [ + "http", + "https", + "ftp", + "file", + "jar" + ], + "url": "http:" + } + }, + "resource": { + "reload": { + "enabled": "true", + "interval": { + "high": "5s", + "low": "60s", + "medium": "30s" + } + } + }, + "rest": { + "action": { + "multi": { + "allow_explicit_index": "true" + } + } + }, + "script": { + "aggs": "false", + "auto_reload_enabled": "true", + "cache": { + "expire": "0ms", + "max_size": "100" + }, + "engine": { + "expression": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "true", + "inline.aggs": "true", + "inline.ingest": "true", + "inline.search": "true", + "inline.update": "true", + "stored": "true", + "stored.aggs": "true", + "stored.ingest": "true", + "stored.search": "true", + "stored.update": "true" + }, + "groovy": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "false", + "inline.aggs": "false", + "inline.ingest": "false", + "inline.search": "false", + "inline.update": "false", + "stored": "false", + "stored.aggs": "false", + "stored.ingest": "false", + "stored.search": "false", + "stored.update": "false" + }, + "mustache": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "true", + "inline.aggs": "true", + "inline.ingest": "true", + "inline.search": "true", + "inline.update": "true", + "stored": "true", + "stored.aggs": "true", + "stored.ingest": "true", + "stored.search": "true", + "stored.update": "true" + }, + "painless": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "true", + "inline.aggs": "true", + "inline.ingest": "true", + "inline.search": "true", + "inline.update": "true", + "stored": "true", + "stored.aggs": "true", + "stored.ingest": "true", + "stored.search": "true", + "stored.update": "true" + } + }, + "file": "true", + "ingest": "false", + "inline": "false", + "legacy": { + "default_lang": "groovy" + }, + "max_compilations_per_minute": "15", + "max_size_in_bytes": "65535", + "painless": { + "regex": { + "enabled": "false" + } + }, + "search": "false", + "stored": "false", + "update": "false" + }, + "search": { + "default_keep_alive": "5m", + "default_search_timeout": "-1", + "highlight": { + "term_vector_multi_value": "true" + }, + "keep_alive_interval": "1m", + "low_level_cancellation": "false", + "remote": { + "connect": "true", + "connections_per_cluster": "3", + "initial_connect_timeout": "30s", + "node": { + "attr": "" + } + } + }, + "security": { + "manager": { + "filter_bad_defaults": "true" + } + }, + "thread_pool": { + "bulk": { + "queue_size": "200", + "size": "4" + }, + "estimated_time_interval": "200ms", + "fetch_shard_started": { + "core": "1", + "keep_alive": "5m", + "max": "8" + }, + "fetch_shard_store": { + "core": "1", + "keep_alive": "5m", + "max": "8" + }, + "flush": { + "core": "1", + "keep_alive": "5m", + "max": "2" + }, + "force_merge": { + "queue_size": "-1", + "size": "1" + }, + "generic": { + "core": "4", + "keep_alive": "30s", + "max": "128" + }, + "get": { + "queue_size": "1000", + "size": "4" + }, + "index": { + "queue_size": "200", + "size": "4" + }, + "listener": { + "queue_size": "-1", + "size": "2" + }, + "management": { + "core": "1", + "keep_alive": "5m", + "max": "5" + }, + "refresh": { + "core": "1", + "keep_alive": "5m", + "max": "2" + }, + "search": { + "queue_size": "1000", + "size": "7" + }, + "snapshot": { + "core": "1", + "keep_alive": "5m", + "max": "2" + }, + "warmer": { + "core": "1", + "keep_alive": "5m", + "max": "2" + } + }, + "transport": { + "connections_per_node": { + "bulk": "3", + "ping": "1", + "recovery": "2", + "reg": "6", + "state": "1" + }, + "netty": { + "boss_count": "1", + "max_composite_buffer_components": "-1", + "max_cumulation_buffer_capacity": "-1b", + "receive_predictor_max": "512kb", + "receive_predictor_min": "512kb", + "receive_predictor_size": "512kb", + "worker_count": "8" + }, + "ping_schedule": "-1", + "publish_port": "-1", + "tcp": { + "blocking_client": "false", + "blocking_server": "false", + "compress": "false", + "connect_timeout": "30s", + "keep_alive": "true", + "port": "9300-9400", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + }, + "tcp_no_delay": "true", + "tracer": { + "exclude": [ + "internal:discovery/zen/fd*", + "cluster:monitor/nodes/liveness" + ] + }, + "type": "", + "type.default": "netty4" + }, + "tribe": { + "blocks": { + "metadata": "false", + "write": "false" + }, + "name": "", + "on_conflict": "any" + } + }, + "persistent": {}, + "transient": {} +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/settings-7.3.0.json b/inputs/elasticsearch/fixtures/settings-7.3.0.json new file mode 100644 index 00000000..f6ca3864 --- /dev/null +++ b/inputs/elasticsearch/fixtures/settings-7.3.0.json @@ -0,0 +1,1096 @@ +{ + "defaults": { + "action": { + "auto_create_index": "true", + "destructive_requires_name": "false", + "search": { + "shard_count": { + "limit": "9223372036854775807" + } + } + }, + "bootstrap": { + "ctrlhandler": "true", + "memory_lock": "false", + "system_call_filter": "true" + }, + "cache": { + "recycler": { + "page": { + "limit": { + "heap": "10%" + }, + "type": "CONCURRENT", + "weight": { + "bytes": "1.0", + "ints": "1.0", + "longs": "1.0", + "objects": "0.1" + } + } + } + }, + "ccr": { + "auto_follow": { + "wait_for_metadata_timeout": "60s" + }, + "indices": { + "recovery": { + "chunk_size": "1mb", + "internal_action_timeout": "60s", + "max_bytes_per_sec": "40mb", + "max_concurrent_file_chunks": "5", + "recovery_activity_timeout": "60s" + } + }, + "wait_for_metadata_timeout": "60s" + }, + "client": { + "transport": { + "ignore_cluster_name": "false", + "nodes_sampler_interval": "5s", + "ping_timeout": "5s", + "sniff": "false" + }, + "type": "node" + }, + "cluster": { + "auto_shrink_voting_configuration": "true", + "blocks": { + "read_only": "false", + "read_only_allow_delete": "false" + }, + "election": { + "back_off_time": "100ms", + "duration": "500ms", + "initial_timeout": "100ms", + "max_timeout": "10s", + "strategy": "supports_voting_only" + }, + "fault_detection": { + "follower_check": { + "interval": "1000ms", + "retry_count": "3", + "timeout": "10000ms" + }, + "leader_check": { + "interval": "1000ms", + "retry_count": "3", + "timeout": "10000ms" + } + }, + "follower_lag": { + "timeout": "90000ms" + }, + "indices": { + "close": { + "enable": "true" + }, + "tombstones": { + "size": "500" + } + }, + "info": { + "update": { + "interval": "30s", + "timeout": "15s" + } + }, + "initial_master_nodes": [], + "join": { + "timeout": "60000ms" + }, + "max_shards_per_node": "1000", + "max_voting_config_exclusions": "10", + "name": "docker-cluster", + "no_master_block": "write", + "nodes": { + "reconnect_interval": "10s" + }, + "persistent_tasks": { + "allocation": { + "enable": "all", + "recheck_interval": "30s" + } + }, + "publish": { + "timeout": "30000ms" + }, + "remote": { + "connect": "true", + "connections_per_cluster": "3", + "initial_connect_timeout": "30s", + "node": { + "attr": "" + } + }, + "routing": { + "allocation": { + "allow_rebalance": "indices_all_active", + "awareness": { + "attributes": [] + }, + "balance": { + "index": "0.55", + "shard": "0.45", + "threshold": "1.0" + }, + "cluster_concurrent_rebalance": "2", + "disk": { + "include_relocations": "true", + "reroute_interval": "60s", + "threshold_enabled": "false", + "watermark": { + "flood_stage": "0.95", + "high": "0.9", + "low": "0.85" + } + }, + "enable": "all", + "node_concurrent_incoming_recoveries": "2", + "node_concurrent_outgoing_recoveries": "2", + "node_concurrent_recoveries": "2", + "node_initial_primaries_recoveries": "4", + "same_shard": { + "host": "false" + }, + "total_shards_per_node": "-1", + "type": "balanced" + }, + "rebalance": { + "enable": "all" + }, + "use_adaptive_replica_selection": "true" + }, + "service": { + "slow_task_logging_threshold": "30s" + } + }, + "data_frame": { + "task_thread_pool": { + "queue_size": "4", + "size": "4" + } + }, + "discovery": { + "cluster_formation_warning_timeout": "10000ms", + "find_peers_interval": "1000ms", + "initial_state_timeout": "30s", + "request_peers_timeout": "3000ms", + "seed_hosts": [], + "seed_providers": [], + "seed_resolver": { + "max_concurrent_resolvers": "10", + "timeout": "5s" + }, + "type": "single-node", + "unconfigured_bootstrap_timeout": "3s", + "zen": { + "bwc_ping_timeout": "3s", + "commit_timeout": "30s", + "fd": { + "connect_on_network_disconnect": "false", + "ping_interval": "1s", + "ping_retries": "3", + "ping_timeout": "30s", + "register_connection_listener": "true" + }, + "hosts_provider": [], + "join_retry_attempts": "3", + "join_retry_delay": "100ms", + "join_timeout": "60000ms", + "master_election": { + "ignore_non_master_pings": "false", + "wait_for_joins_timeout": "30000ms" + }, + "max_pings_from_another_master": "3", + "minimum_master_nodes": "-1", + "no_master_block": "write", + "ping": { + "unicast": { + "concurrent_connects": "10", + "hosts": [], + "hosts.resolve_timeout": "5s" + } + }, + "ping_timeout": "3s", + "publish": { + "max_pending_cluster_states": "25" + }, + "publish_diff": { + "enable": "true" + }, + "publish_timeout": "30s", + "send_leave_request": "true", + "unsafe_rolling_upgrades_enabled": "true" + } + }, + "gateway": { + "expected_data_nodes": "-1", + "expected_master_nodes": "-1", + "expected_nodes": "-1", + "recover_after_data_nodes": "-1", + "recover_after_master_nodes": "0", + "recover_after_nodes": "-1", + "recover_after_time": "0ms" + }, + "http": { + "bind_host": [], + "compression": "true", + "compression_level": "3", + "content_type": { + "required": "true" + }, + "cors": { + "allow-credentials": "false", + "allow-headers": "X-Requested-With,Content-Type,Content-Length", + "allow-methods": "OPTIONS,HEAD,GET,POST,PUT,DELETE", + "allow-origin": "", + "enabled": "false", + "max-age": "1728000" + }, + "detailed_errors": { + "enabled": "true" + }, + "host": [], + "max_chunk_size": "8kb", + "max_content_length": "100mb", + "max_header_size": "8kb", + "max_initial_line_length": "4kb", + "max_warning_header_count": "-1", + "max_warning_header_size": "-1b", + "netty": { + "max_composite_buffer_components": "69905", + "receive_predictor_size": "64kb", + "worker_count": "16" + }, + "pipelining": { + "max_events": "10000" + }, + "port": "9200-9300", + "publish_host": [], + "publish_port": "-1", + "read_timeout": "0ms", + "reset_cookies": "false", + "tcp": { + "keep_alive": "true", + "no_delay": "true", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + }, + "tcp_no_delay": "true", + "type": "security4", + "type.default": "netty4" + }, + "index": { + "codec": "default", + "store": { + "fs": { + "fs_lock": "native" + }, + "preload": [], + "type": "" + } + }, + "indices": { + "analysis": { + "hunspell": { + "dictionary": { + "ignore_case": "false", + "lazy": "false" + } + } + }, + "breaker": { + "accounting": { + "limit": "100%", + "overhead": "1.0" + }, + "fielddata": { + "limit": "40%", + "overhead": "1.03", + "type": "memory" + }, + "request": { + "limit": "60%", + "overhead": "1.0", + "type": "memory" + }, + "total": { + "limit": "95%", + "use_real_memory": "true" + }, + "type": "hierarchy" + }, + "cache": { + "cleanup_interval": "1m" + }, + "fielddata": { + "cache": { + "size": "-1b" + } + }, + "lifecycle": { + "poll_interval": "10m" + }, + "mapping": { + "dynamic_timeout": "30s" + }, + "memory": { + "index_buffer_size": "10%", + "interval": "5s", + "max_index_buffer_size": "-1", + "min_index_buffer_size": "48mb", + "shard_inactive_time": "5m" + }, + "queries": { + "cache": { + "all_segments": "false", + "count": "10000", + "size": "10%" + } + }, + "query": { + "bool": { + "max_clause_count": "1024" + }, + "query_string": { + "allowLeadingWildcard": "true", + "analyze_wildcard": "false" + } + }, + "recovery": { + "internal_action_long_timeout": "1800000ms", + "internal_action_timeout": "15m", + "max_bytes_per_sec": "40mb", + "max_concurrent_file_chunks": "2", + "recovery_activity_timeout": "1800000ms", + "retry_delay_network": "5s", + "retry_delay_state_sync": "500ms" + }, + "requests": { + "cache": { + "expire": "0ms", + "size": "1%" + } + }, + "store": { + "delete": { + "shard": { + "timeout": "30s" + } + } + } + }, + "ingest": { + "geoip": { + "cache_size": "1000" + }, + "grok": { + "watchdog": { + "interval": "1s", + "max_execution_time": "1s" + } + } + }, + "logger": { + "level": "INFO" + }, + "monitor": { + "fs": { + "refresh_interval": "1s" + }, + "jvm": { + "gc": { + "enabled": "true", + "overhead": { + "debug": "10", + "info": "25", + "warn": "50" + }, + "refresh_interval": "1s" + }, + "refresh_interval": "1s" + }, + "os": { + "refresh_interval": "1s" + }, + "process": { + "refresh_interval": "1s" + } + }, + "network": { + "bind_host": [ + "0.0.0.0" + ], + "breaker": { + "inflight_requests": { + "limit": "100%", + "overhead": "2.0" + } + }, + "host": [ + "0.0.0.0" + ], + "publish_host": [ + "0.0.0.0" + ], + "server": "true", + "tcp": { + "connect_timeout": "30s", + "keep_alive": "true", + "no_delay": "true", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + } + }, + "no": { + "model": { + "state": { + "persist": "false" + } + } + }, + "node": { + "attr": { + "ml": { + "machine_memory": "8255340544", + "max_open_jobs": "20" + }, + "xpack": { + "installed": "true" + } + }, + "data": "true", + "enable_lucene_segment_infos_trace": "false", + "id": { + "seed": "0" + }, + "ingest": "true", + "local_storage": "true", + "master": "true", + "max_local_storage_nodes": "1", + "ml": "true", + "name": "2c26cd7c415b", + "portsfile": "false", + "store": { + "allow_mmap": "true" + }, + "voting_only": "false" + }, + "path": { + "data": [], + "home": "/usr/share/elasticsearch", + "logs": "/usr/share/elasticsearch/logs", + "repo": [], + "shared_data": "" + }, + "pidfile": "", + "plugin": { + "mandatory": [] + }, + "processors": "8", + "reindex": { + "remote": { + "whitelist": [] + } + }, + "repositories": { + "fs": { + "chunk_size": "9223372036854775807b", + "compress": "false", + "location": "" + }, + "url": { + "allowed_urls": [], + "supported_protocols": [ + "http", + "https", + "ftp", + "file", + "jar" + ], + "url": "http:" + } + }, + "resource": { + "reload": { + "enabled": "true", + "interval": { + "high": "5s", + "low": "60s", + "medium": "30s" + } + } + }, + "rest": { + "action": { + "multi": { + "allow_explicit_index": "true" + } + } + }, + "script": { + "allowed_contexts": [], + "allowed_types": [], + "cache": { + "expire": "0ms", + "max_size": "100" + }, + "max_compilations_rate": "75/5m", + "max_size_in_bytes": "65535", + "painless": { + "regex": { + "enabled": "false" + } + } + }, + "search": { + "default_allow_partial_results": "true", + "default_keep_alive": "5m", + "default_search_timeout": "-1", + "highlight": { + "term_vector_multi_value": "true" + }, + "keep_alive_interval": "1m", + "low_level_cancellation": "true", + "max_buckets": "10000", + "max_keep_alive": "24h", + "max_open_scroll_context": "500", + "remote": { + "connect": "true", + "connections_per_cluster": "3", + "initial_connect_timeout": "30s", + "node": { + "attr": "" + } + } + }, + "security": { + "manager": { + "filter_bad_defaults": "true" + } + }, + "thread_pool": { + "analyze": { + "queue_size": "16", + "size": "1" + }, + "estimated_time_interval": "200ms", + "fetch_shard_started": { + "core": "1", + "keep_alive": "5m", + "max": "16" + }, + "fetch_shard_store": { + "core": "1", + "keep_alive": "5m", + "max": "16" + }, + "flush": { + "core": "1", + "keep_alive": "5m", + "max": "4" + }, + "force_merge": { + "queue_size": "-1", + "size": "1" + }, + "generic": { + "core": "4", + "keep_alive": "30s", + "max": "128" + }, + "get": { + "queue_size": "1000", + "size": "8" + }, + "listener": { + "queue_size": "-1", + "size": "4" + }, + "management": { + "core": "1", + "keep_alive": "5m", + "max": "5" + }, + "refresh": { + "core": "1", + "keep_alive": "5m", + "max": "4" + }, + "search": { + "auto_queue_frame_size": "2000", + "max_queue_size": "1000", + "min_queue_size": "1000", + "queue_size": "1000", + "size": "13", + "target_response_time": "1s" + }, + "search_throttled": { + "auto_queue_frame_size": "200", + "max_queue_size": "100", + "min_queue_size": "100", + "queue_size": "100", + "size": "1", + "target_response_time": "1s" + }, + "snapshot": { + "core": "1", + "keep_alive": "5m", + "max": "4" + }, + "warmer": { + "core": "1", + "keep_alive": "5m", + "max": "4" + }, + "write": { + "queue_size": "200", + "size": "8" + } + }, + "transport": { + "bind_host": [], + "compress": "false", + "connect_timeout": "30s", + "connections_per_node": { + "bulk": "3", + "ping": "1", + "recovery": "2", + "reg": "6", + "state": "1" + }, + "features": { + "x-pack": "true" + }, + "host": [], + "netty": { + "boss_count": "1", + "receive_predictor_max": "64kb", + "receive_predictor_min": "64kb", + "receive_predictor_size": "64kb", + "worker_count": "16" + }, + "ping_schedule": "-1", + "port": "9300-9400", + "publish_host": [], + "publish_port": "-1", + "tcp": { + "compress": "false", + "connect_timeout": "30s", + "keep_alive": "true", + "no_delay": "true", + "port": "9300-9400", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + }, + "tcp_no_delay": "true", + "tracer": { + "exclude": [ + "internal:discovery/zen/fd*", + "internal:coordination/fault_detection/*", + "cluster:monitor/nodes/liveness" + ], + "include": [] + }, + "type": "security4", + "type.default": "netty4" + }, + "xpack": { + "ccr": { + "ccr_thread_pool": { + "queue_size": "100", + "size": "32" + }, + "enabled": "true" + }, + "data_frame": { + "enabled": "true" + }, + "flattened": { + "enabled": "true" + }, + "graph": { + "enabled": "true" + }, + "http": { + "default_connection_timeout": "10s", + "default_read_timeout": "10s", + "max_response_size": "10mb", + "proxy": { + "host": "", + "port": "0", + "scheme": "" + }, + "whitelist": [ + "*" + ] + }, + "ilm": { + "enabled": "true" + }, + "license": { + "self_generated": { + "type": "basic" + } + }, + "logstash": { + "enabled": "true" + }, + "ml": { + "autodetect_process": "true", + "datafeed_thread_pool": { + "core": "1", + "keep_alive": "1m", + "max": "512" + }, + "enable_config_migration": "true", + "enabled": "true", + "job_comms_thread_pool": { + "core": "4", + "keep_alive": "1m", + "max": "2048" + }, + "max_anomaly_records": "500", + "max_lazy_ml_nodes": "0", + "max_machine_memory_percent": "30", + "max_model_memory_limit": "0b", + "max_open_jobs": "20", + "min_disk_space_off_heap": "5gb", + "node_concurrent_job_allocations": "2", + "process_connect_timeout": "10s", + "utility_thread_pool": { + "core": "1", + "keep_alive": "10m", + "max": "2048" + } + }, + "monitoring": { + "collection": { + "ccr": { + "stats": { + "timeout": "10s" + } + }, + "cluster": { + "stats": { + "timeout": "10s" + } + }, + "enabled": "false", + "index": { + "recovery": { + "active_only": "false", + "timeout": "10s" + }, + "stats": { + "timeout": "10s" + } + }, + "indices": [], + "interval": "10s", + "ml": { + "job": { + "stats": { + "timeout": "10s" + } + } + }, + "node": { + "stats": { + "timeout": "10s" + } + } + }, + "elasticsearch": { + "collection": { + "enabled": "true" + } + }, + "enabled": "true", + "history": { + "duration": "168h" + } + }, + "notification": { + "email": { + "default_account": "", + "html": { + "sanitization": { + "allow": [ + "body", + "head", + "_tables", + "_links", + "_blocks", + "_formatting", + "img:embedded" + ], + "disallow": [], + "enabled": "true" + } + } + }, + "jira": { + "default_account": "" + }, + "pagerduty": { + "default_account": "" + }, + "reporting": { + "interval": "15s", + "retries": "40" + }, + "slack": { + "default_account": "" + } + }, + "rollup": { + "enabled": "true", + "task_thread_pool": { + "queue_size": "4", + "size": "4" + } + }, + "security": { + "audit": { + "enabled": "false", + "logfile": { + "emit_node_host_address": "false", + "emit_node_host_name": "false", + "emit_node_id": "true", + "emit_node_name": "false", + "events": { + "emit_request_body": "false", + "exclude": [], + "include": [ + "ACCESS_DENIED", + "ACCESS_GRANTED", + "ANONYMOUS_ACCESS_DENIED", + "AUTHENTICATION_FAILED", + "CONNECTION_DENIED", + "TAMPERED_REQUEST", + "RUN_AS_DENIED", + "RUN_AS_GRANTED" + ] + } + } + }, + "authc": { + "anonymous": { + "authz_exception": "true", + "roles": [], + "username": "_anonymous" + }, + "api_key": { + "cache": { + "hash_algo": "ssha256", + "max_keys": "10000", + "ttl": "24h" + }, + "delete": { + "interval": "24h", + "timeout": "-1" + }, + "enabled": "false", + "hashing": { + "algorithm": "pbkdf2" + } + }, + "password_hashing": { + "algorithm": "bcrypt" + }, + "reserved_realm": { + "enabled": "true" + }, + "run_as": { + "enabled": "true" + }, + "success_cache": { + "enabled": "true", + "expire_after_access": "1h", + "size": "10000" + }, + "token": { + "delete": { + "interval": "30m", + "timeout": "-1" + }, + "enabled": "false", + "thread_pool": { + "queue_size": "1000", + "size": "1" + }, + "timeout": "20m" + } + }, + "authz": { + "store": { + "roles": { + "cache": { + "max_size": "10000" + }, + "field_permissions": { + "cache": { + "max_size_in_bytes": "104857600" + } + }, + "index": { + "cache": { + "max_size": "10000", + "ttl": "20m" + } + }, + "negative_lookup_cache": { + "max_size": "10000" + } + } + } + }, + "automata": { + "cache": { + "enabled": "true", + "size": "10000", + "ttl": "48h" + }, + "max_determinized_states": "100000" + }, + "dls": { + "bitset": { + "cache": { + "size": "50mb", + "ttl": "168h" + } + } + }, + "dls_fls": { + "enabled": "true" + }, + "enabled": "true", + "encryption": { + "algorithm": "AES/CTR/NoPadding" + }, + "encryption_key": { + "algorithm": "AES", + "length": "128" + }, + "filter": { + "always_allow_bound_address": "true" + }, + "fips_mode": { + "enabled": "false" + }, + "http": { + "filter": { + "allow": [], + "deny": [], + "enabled": "true" + }, + "ssl": { + "enabled": "false" + } + }, + "transport": { + "filter": { + "allow": [], + "deny": [], + "enabled": "true" + }, + "ssl": { + "enabled": "false" + } + }, + "user": null + }, + "sql": { + "enabled": "true" + }, + "vectors": { + "enabled": "true" + }, + "watcher": { + "actions": { + "bulk": { + "default_timeout": "" + }, + "index": { + "default_timeout": "" + } + }, + "bulk": { + "actions": "1", + "concurrent_requests": "0", + "flush_interval": "1s", + "size": "1mb" + }, + "enabled": "true", + "encrypt_sensitive_data": "false", + "execution": { + "default_throttle_period": "5s", + "scroll": { + "size": "0", + "timeout": "" + } + }, + "history": { + "cleaner_service": { + "enabled": "true" + } + }, + "index": { + "rest": { + "direct_access": "" + } + }, + "input": { + "search": { + "default_timeout": "" + } + }, + "internal": { + "ops": { + "bulk": { + "default_timeout": "" + }, + "index": { + "default_timeout": "" + }, + "search": { + "default_timeout": "" + } + } + }, + "stop": { + "timeout": "30s" + }, + "thread_pool": { + "queue_size": "1000", + "size": "40" + }, + "transform": { + "search": { + "default_timeout": "" + } + }, + "trigger": { + "schedule": { + "ticker": { + "tick_interval": "500ms" + } + } + }, + "watch": { + "scroll": { + "size": "0" + } + } + } + } + }, + "persistent": {}, + "transient": {} +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/settings-merge-5.4.2.json b/inputs/elasticsearch/fixtures/settings-merge-5.4.2.json new file mode 100644 index 00000000..edb49e1f --- /dev/null +++ b/inputs/elasticsearch/fixtures/settings-merge-5.4.2.json @@ -0,0 +1,668 @@ +{ + "defaults": { + "action": { + "auto_create_index": "true", + "destructive_requires_name": "false", + "master": { + "force_local": "false" + }, + "search": { + "shard_count": { + "limit": "9223372036854775807" + } + } + }, + "bootstrap": { + "ctrlhandler": "true", + "memory_lock": "false", + "seccomp": "true", + "system_call_filter": "true" + }, + "cache": { + "recycler": { + "page": { + "limit": { + "heap": "10%" + }, + "type": "CONCURRENT", + "weight": { + "bytes": "1.0", + "ints": "1.0", + "longs": "1.0", + "objects": "0.1" + } + } + } + }, + "client": { + "transport": { + "ignore_cluster_name": "false", + "nodes_sampler_interval": "5s", + "ping_timeout": "5s", + "sniff": "false" + }, + "type": "node" + }, + "cluster": { + "blocks": { + "read_only": "false" + }, + "indices": { + "close": { + "enable": "true" + } + }, + "info": { + "update": { + "interval": "30s", + "timeout": "15s" + } + }, + "name": "elasticsearch", + "nodes": { + "reconnect_interval": "10s" + }, + "routing": { + "none": { + "allow_rebalance": "indices_all_active", + "awareness": { + "attributes": "" + }, + "balance": { + "index": "0.55", + "shard": "0.45", + "threshold": "1.0" + }, + "cluster_concurrent_rebalance": "2", + "disk": { + "include_relocations": "true", + "reroute_interval": "60s", + "threshold_enabled": "true", + "watermark": { + "high": "90%", + "low": "85%" + } + }, + "node_concurrent_incoming_recoveries": "2", + "node_concurrent_outgoing_recoveries": "2", + "node_concurrent_recoveries": "2", + "node_initial_primaries_recoveries": "4", + "same_shard": { + "host": "false" + }, + "snapshot": { + "relocation_enabled": "false" + }, + "total_shards_per_node": "-1", + "type": "balanced" + }, + "rebalance": { + "enable": "ALL" + } + }, + "service": { + "slow_task_logging_threshold": "30s" + } + }, + "default": { + "path": { + "conf": "", + "logs": "" + } + }, + "discovery": { + "initial_state_timeout": "30s", + "type": "zen", + "zen": { + "commit_timeout": "30s", + "fd": { + "connect_on_network_disconnect": "false", + "ping_interval": "1s", + "ping_retries": "3", + "ping_timeout": "30s", + "register_connection_listener": "true" + }, + "hosts_provider": null, + "join_retry_attempts": "3", + "join_retry_delay": "100ms", + "join_timeout": "60000ms", + "master_election": { + "ignore_non_master_pings": "false", + "wait_for_joins_timeout": "30000ms" + }, + "max_pings_from_another_master": "3", + "minimum_master_nodes": "-1", + "no_master_block": "write", + "ping": { + "unicast": { + "concurrent_connects": "10", + "hosts": { + "resolve_timeout": "5s" + } + } + }, + "ping_timeout": "3s", + "publish_diff": { + "enable": "true" + }, + "publish_timeout": "30s", + "send_leave_request": "true" + } + }, + "gateway": { + "expected_data_nodes": "-1", + "expected_master_nodes": "-1", + "expected_nodes": "-1", + "initial_shards": "quorum", + "recover_after_data_nodes": "-1", + "recover_after_master_nodes": "0", + "recover_after_nodes": "-1", + "recover_after_time": "0ms" + }, + "http": { + "bind_host": [ + "0.0.0.0" + ], + "compression": "true", + "compression_level": "3", + "content_type": { + "required": "false" + }, + "cors": { + "allow-credentials": "false", + "allow-headers": "X-Requested-With,Content-Type,Content-Length", + "allow-methods": "OPTIONS,HEAD,GET,POST,PUT,DELETE", + "allow-origin": "", + "enabled": "false", + "max-age": "1728000" + }, + "detailed_errors": { + "enabled": "true" + }, + "enabled": "true", + "host": [ + "0.0.0.0" + ], + "max_chunk_size": "8kb", + "max_content_length": "100mb", + "max_header_size": "8kb", + "max_initial_line_length": "4kb", + "netty": { + "max_composite_buffer_components": "-1", + "max_cumulation_buffer_capacity": "-1b", + "receive_predictor_max": "64kb", + "receive_predictor_min": "64kb", + "receive_predictor_size": "64kb", + "worker_count": "8" + }, + "pipelining": "true", + "pipelining.max_events": "10000", + "port": "9200-9300", + "publish_host": [ + "0.0.0.0" + ], + "publish_port": "-1", + "reset_cookies": "false", + "tcp": { + "blocking_server": "false", + "keep_alive": "true", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + }, + "tcp_no_delay": "true", + "type": "", + "type.default": "netty4" + }, + "index": { + "codec": "default", + "store": { + "fs": { + "fs_lock": "native" + }, + "type": "" + } + }, + "indices": { + "analysis": { + "hunspell": { + "dictionary": { + "ignore_case": "false", + "lazy": "false" + } + } + }, + "breaker": { + "fielddata": { + "limit": "60%", + "overhead": "1.03", + "type": "memory" + }, + "request": { + "limit": "60%", + "overhead": "1.0", + "type": "memory" + }, + "total": { + "limit": "70%" + }, + "type": "hierarchy" + }, + "cache": { + "cleanup_interval": "1m" + }, + "fielddata": { + "cache": { + "size": "-1b" + } + }, + "mapping": { + "dynamic_timeout": "30s" + }, + "memory": { + "index_buffer_size": "10%", + "interval": "5s", + "max_index_buffer_size": "-1b", + "min_index_buffer_size": "48mb", + "shard_inactive_time": "5m" + }, + "queries": { + "cache": { + "all_segments": "false", + "count": "10000", + "size": "10%" + } + }, + "query": { + "bool": { + "max_clause_count": "1024" + }, + "query_string": { + "allowLeadingWildcard": "true", + "analyze_wildcard": "false" + } + }, + "recovery": { + "internal_action_long_timeout": "1800000ms", + "internal_action_timeout": "15m", + "max_bytes_per_sec": "40mb", + "recovery_activity_timeout": "1800000ms", + "retry_delay_network": "5s", + "retry_delay_state_sync": "500ms" + }, + "requests": { + "cache": { + "expire": "0ms", + "size": "1%" + } + }, + "store": { + "delete": { + "shard": { + "timeout": "30s" + } + }, + "throttle": { + "max_bytes_per_sec": "0b", + "type": "NONE" + } + }, + "ttl": { + "interval": "60s" + } + }, + "logger": { + "level": "INFO" + }, + "monitor": { + "fs": { + "refresh_interval": "1s" + }, + "jvm": { + "gc": { + "enabled": "true", + "overhead": { + "debug": "10", + "info": "25", + "warn": "50" + }, + "refresh_interval": "1s" + }, + "refresh_interval": "1s" + }, + "os": { + "refresh_interval": "1s" + }, + "process": { + "refresh_interval": "1s" + } + }, + "network": { + "bind_host": [ + "_local_" + ], + "breaker": { + "inflight_requests": { + "limit": "100%", + "overhead": "1.0" + } + }, + "host": [ + "_local_" + ], + "publish_host": [ + "_local_" + ], + "server": "true", + "tcp": { + "blocking": "false", + "blocking_client": "false", + "blocking_server": "false", + "connect_timeout": "30s", + "keep_alive": "true", + "no_delay": "true", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + } + }, + "node": { + "add_lock_id_to_custom_path": "true", + "data": "true", + "enable_lucene_segment_infos_trace": "false", + "id": { + "seed": "0" + }, + "ingest": "true", + "local_storage": "true", + "master": "true", + "max_local_storage_nodes": "1", + "name": "DK8-2Lc", + "portsfile": "false" + }, + "path": { + "conf": "", + "home": "/usr/share/elasticsearch", + "logs": "/usr/share/elasticsearch/logs", + "scripts": "", + "shared_data": "" + }, + "pidfile": "", + "processors": "4", + "repositories": { + "fs": { + "chunk_size": "-1b", + "compress": "false", + "location": "" + }, + "url": { + "supported_protocols": [ + "http", + "https", + "ftp", + "file", + "jar" + ], + "url": "http:" + } + }, + "resource": { + "reload": { + "enabled": "true", + "interval": { + "high": "5s", + "low": "60s", + "medium": "30s" + } + } + }, + "rest": { + "action": { + "multi": { + "allow_explicit_index": "true" + } + } + }, + "script": { + "aggs": "false", + "auto_reload_enabled": "true", + "cache": { + "expire": "0ms", + "max_size": "100" + }, + "engine": { + "expression": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "true", + "inline.aggs": "true", + "inline.ingest": "true", + "inline.search": "true", + "inline.update": "true", + "stored": "true", + "stored.aggs": "true", + "stored.ingest": "true", + "stored.search": "true", + "stored.update": "true" + }, + "groovy": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "false", + "inline.aggs": "false", + "inline.ingest": "false", + "inline.search": "false", + "inline.update": "false", + "stored": "false", + "stored.aggs": "false", + "stored.ingest": "false", + "stored.search": "false", + "stored.update": "false" + }, + "mustache": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "true", + "inline.aggs": "true", + "inline.ingest": "true", + "inline.search": "true", + "inline.update": "true", + "stored": "true", + "stored.aggs": "true", + "stored.ingest": "true", + "stored.search": "true", + "stored.update": "true" + }, + "painless": { + "file": "true", + "file.aggs": "true", + "file.ingest": "true", + "file.search": "true", + "file.update": "true", + "inline": "true", + "inline.aggs": "true", + "inline.ingest": "true", + "inline.search": "true", + "inline.update": "true", + "stored": "true", + "stored.aggs": "true", + "stored.ingest": "true", + "stored.search": "true", + "stored.update": "true" + } + }, + "file": "true", + "ingest": "false", + "inline": "false", + "legacy": { + "default_lang": "groovy" + }, + "max_compilations_per_minute": "15", + "max_size_in_bytes": "65535", + "painless": { + "regex": { + "enabled": "false" + } + }, + "search": "false", + "stored": "false", + "update": "false" + }, + "search": { + "default_keep_alive": "5m", + "default_search_timeout": "-1", + "highlight": { + "term_vector_multi_value": "true" + }, + "keep_alive_interval": "1m", + "low_level_cancellation": "false", + "remote": { + "connect": "true", + "connections_per_cluster": "3", + "initial_connect_timeout": "30s", + "node": { + "attr": "" + } + } + }, + "security": { + "manager": { + "filter_bad_defaults": "true" + } + }, + "thread_pool": { + "bulk": { + "queue_size": "200", + "size": "4" + }, + "estimated_time_interval": "200ms", + "fetch_shard_started": { + "core": "1", + "keep_alive": "5m", + "max": "8" + }, + "fetch_shard_store": { + "core": "1", + "keep_alive": "5m", + "max": "8" + }, + "flush": { + "core": "1", + "keep_alive": "5m", + "max": "2" + }, + "force_merge": { + "queue_size": "-1", + "size": "1" + }, + "generic": { + "core": "4", + "keep_alive": "30s", + "max": "128" + }, + "get": { + "queue_size": "1000", + "size": "4" + }, + "index": { + "queue_size": "200", + "size": "4" + }, + "listener": { + "queue_size": "-1", + "size": "2" + }, + "management": { + "core": "1", + "keep_alive": "5m", + "max": "5" + }, + "refresh": { + "core": "1", + "keep_alive": "5m", + "max": "2" + }, + "search": { + "queue_size": "1000", + "size": "7" + }, + "snapshot": { + "core": "1", + "keep_alive": "5m", + "max": "2" + }, + "warmer": { + "core": "1", + "keep_alive": "5m", + "max": "2" + } + }, + "transport": { + "connections_per_node": { + "bulk": "3", + "ping": "1", + "recovery": "2", + "reg": "6", + "state": "1" + }, + "netty": { + "boss_count": "1", + "max_composite_buffer_components": "-1", + "max_cumulation_buffer_capacity": "-1b", + "receive_predictor_max": "512kb", + "receive_predictor_min": "512kb", + "receive_predictor_size": "512kb", + "worker_count": "8" + }, + "ping_schedule": "-1", + "publish_port": "-1", + "tcp": { + "blocking_client": "false", + "blocking_server": "false", + "compress": "false", + "connect_timeout": "30s", + "keep_alive": "true", + "port": "9300-9400", + "receive_buffer_size": "-1b", + "reuse_address": "true", + "send_buffer_size": "-1b" + }, + "tcp_no_delay": "true", + "tracer": { + "exclude": [ + "internal:discovery/zen/fd*", + "cluster:monitor/nodes/liveness" + ] + }, + "type": "", + "type.default": "netty4" + }, + "tribe": { + "blocks": { + "metadata": "false", + "write": "false" + }, + "name": "", + "on_conflict": "any" + } + }, + "persistent": {}, + "transient": { + "cluster": { + "routing": { + "allocation": { + "enable": "ALL" + } + } + } + } +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/snapshots/1.7.6.json b/inputs/elasticsearch/fixtures/snapshots/1.7.6.json new file mode 100644 index 00000000..b32bcb10 --- /dev/null +++ b/inputs/elasticsearch/fixtures/snapshots/1.7.6.json @@ -0,0 +1,25 @@ +{ + "snapshots": [ + { + "duration_in_millis": 328, + "end_time": "2018-09-04T09:09:02.755Z", + "end_time_in_millis": 1536052142755, + "failures": [], + "indices": [ + "foo_1", + "foo_2" + ], + "shards": { + "failed": 0, + "successful": 10, + "total": 10 + }, + "snapshot": "snapshot_1", + "start_time": "2018-09-04T09:09:02.427Z", + "start_time_in_millis": 1536052142427, + "state": "SUCCESS", + "version": "1.7.6", + "version_id": 1070699 + } + ] +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/snapshots/2.4.5.json b/inputs/elasticsearch/fixtures/snapshots/2.4.5.json new file mode 100644 index 00000000..b471f88c --- /dev/null +++ b/inputs/elasticsearch/fixtures/snapshots/2.4.5.json @@ -0,0 +1,25 @@ +{ + "snapshots": [ + { + "duration_in_millis": 508, + "end_time": "2018-09-04T09:25:26.326Z", + "end_time_in_millis": 1536053126326, + "failures": [], + "indices": [ + "foo_2", + "foo_1" + ], + "shards": { + "failed": 0, + "successful": 10, + "total": 10 + }, + "snapshot": "snapshot_1", + "start_time": "2018-09-04T09:25:25.818Z", + "start_time_in_millis": 1536053125818, + "state": "SUCCESS", + "version": "2.4.5", + "version_id": 2040599 + } + ] +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/snapshots/5.4.2-failed.json b/inputs/elasticsearch/fixtures/snapshots/5.4.2-failed.json new file mode 100644 index 00000000..50a9e284 --- /dev/null +++ b/inputs/elasticsearch/fixtures/snapshots/5.4.2-failed.json @@ -0,0 +1,35 @@ +{ + "snapshots": [ + { + "duration_in_millis": 506, + "end_time": "2018-09-04T09:29:14.477Z", + "end_time_in_millis": 1536053354477, + "failures": [ + { + "index": "index_name", + "index_uuid": "index_name", + "node_id": "pPm9jafyTjyMk0T5A101xA", + "reason": "IndexShardSnapshotFailedException[error deleting index file [pending-index-5] during cleanup]; nested: NoSuchFileException[Blob [pending-index-5] does not exist]; ", + "shard_id": 52, + "status": "INTERNAL_SERVER_ERROR" + } + ], + "indices": [ + "foo_2", + "foo_1" + ], + "shards": { + "failed": 1, + "successful": 10, + "total": 10 + }, + "snapshot": "snapshot_1", + "start_time": "2018-09-04T09:29:13.971Z", + "start_time_in_millis": 1536053353971, + "state": "SUCCESS", + "uuid": "VZ_c_kKISAW8rpcqiwSg0w", + "version": "5.4.2", + "version_id": 5040299 + } + ] +} \ No newline at end of file diff --git a/inputs/elasticsearch/fixtures/snapshots/5.4.2.json b/inputs/elasticsearch/fixtures/snapshots/5.4.2.json new file mode 100644 index 00000000..93d94402 --- /dev/null +++ b/inputs/elasticsearch/fixtures/snapshots/5.4.2.json @@ -0,0 +1,26 @@ +{ + "snapshots": [ + { + "duration_in_millis": 506, + "end_time": "2018-09-04T09:29:14.477Z", + "end_time_in_millis": 1536053354477, + "failures": [], + "indices": [ + "foo_2", + "foo_1" + ], + "shards": { + "failed": 0, + "successful": 10, + "total": 10 + }, + "snapshot": "snapshot_1", + "start_time": "2018-09-04T09:29:13.971Z", + "start_time_in_millis": 1536053353971, + "state": "SUCCESS", + "uuid": "VZ_c_kKISAW8rpcqiwSg0w", + "version": "5.4.2", + "version_id": 5040299 + } + ] +} \ No newline at end of file diff --git a/inputs/elasticsearch/pkg/clusterinfo/clusterinfo.go b/inputs/elasticsearch/pkg/clusterinfo/clusterinfo.go new file mode 100644 index 00000000..ef7aa4e4 --- /dev/null +++ b/inputs/elasticsearch/pkg/clusterinfo/clusterinfo.go @@ -0,0 +1,267 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterinfo + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + namespace = "elasticsearch" + subsystem = "clusterinfo" +) + +var ( + // ErrConsumerAlreadyRegistered is returned if a consumer is already registered + ErrConsumerAlreadyRegistered = errors.New("consumer already registered") + // ErrInitialCallTimeout is returned if the initial clusterinfo call timed out + ErrInitialCallTimeout = errors.New("initial cluster info call timed out") + initialTimeout = 10 * time.Second +) + +type consumer interface { + // ClusterLabelUpdates returns a pointer to channel for cluster label updates + ClusterLabelUpdates() *chan *Response + // String implements the stringer interface + String() string +} + +// Retriever periodically gets the cluster info from the / endpoint and +// sends it to all registered consumer channels +type Retriever struct { + consumerChannels map[string]*chan *Response + client *http.Client + url *url.URL + interval time.Duration + sync chan struct{} + versionMetric *prometheus.GaugeVec + up *prometheus.GaugeVec + lastUpstreamSuccessTs *prometheus.GaugeVec + lastUpstreamErrorTs *prometheus.GaugeVec +} + +// New creates a new Retriever +func New(client *http.Client, u *url.URL, interval time.Duration) *Retriever { + return &Retriever{ + consumerChannels: make(map[string]*chan *Response), + client: client, + url: u, + interval: interval, + sync: make(chan struct{}, 1), + versionMetric: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, subsystem, "version_info"), + Help: "Constant metric with ES version information as labels", + }, + []string{ + "cluster", + "cluster_uuid", + "build_date", + "build_hash", + "version", + "lucene_version", + }, + ), + up: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, subsystem, "up"), + Help: "Up metric for the cluster info collector", + }, + []string{"url"}, + ), + lastUpstreamSuccessTs: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, subsystem, "last_retrieval_success_ts"), + Help: "Timestamp of the last successful cluster info retrieval", + }, + []string{"url"}, + ), + lastUpstreamErrorTs: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, subsystem, "last_retrieval_failure_ts"), + Help: "Timestamp of the last failed cluster info retrieval", + }, + []string{"url"}, + ), + } +} + +// Describe implements the prometheus.Collector interface +func (r *Retriever) Describe(ch chan<- *prometheus.Desc) { + r.versionMetric.Describe(ch) + r.up.Describe(ch) + r.lastUpstreamSuccessTs.Describe(ch) + r.lastUpstreamErrorTs.Describe(ch) +} + +// Collect implements the prometheus.Collector interface +func (r *Retriever) Collect(ch chan<- prometheus.Metric) { + r.versionMetric.Collect(ch) + r.up.Collect(ch) + r.lastUpstreamSuccessTs.Collect(ch) + r.lastUpstreamErrorTs.Collect(ch) +} + +func (r *Retriever) updateMetrics(res *Response) { + u := *r.url + u.User = nil + url := u.String() + // scrape failed, response is nil + if res == nil { + r.up.WithLabelValues(url).Set(0.0) + r.lastUpstreamErrorTs.WithLabelValues(url).Set(float64(time.Now().Unix())) + return + } + r.up.WithLabelValues(url).Set(1.0) + r.versionMetric.WithLabelValues( + res.ClusterName, + res.ClusterUUID, + res.Version.BuildDate, + res.Version.BuildHash, + res.Version.Number.String(), + res.Version.LuceneVersion.String(), + ).Set(1.0) + r.lastUpstreamSuccessTs.WithLabelValues(url).Set(float64(time.Now().Unix())) +} + +// Update triggers an external cluster info label update +func (r *Retriever) Update() { + r.sync <- struct{}{} +} + +// RegisterConsumer registers a consumer for cluster info updates +func (r *Retriever) RegisterConsumer(c consumer) error { + if _, registered := r.consumerChannels[c.String()]; registered { + return ErrConsumerAlreadyRegistered + } + r.consumerChannels[c.String()] = c.ClusterLabelUpdates() + return nil +} + +// Run starts the update loop and periodically queries the / endpoint +// The update loop is terminated upon ctx cancellation. The call blocks until the first +// call to the cluster info endpoint was successful +func (r *Retriever) Run(ctx context.Context) error { + startupComplete := make(chan struct{}) + // start update routine + go func(ctx context.Context) { + for { + select { + case <-ctx.Done(): + log.Println("context cancelled, exiting cluster info update loop, err: ", ctx.Err()) + return + case <-r.sync: + log.Println("providing consumers with updated cluster info label") + res, err := r.fetchAndDecodeClusterInfo() + if err != nil { + log.Println("failed to retrieve cluster info from ES, err: ", err) + r.updateMetrics(nil) + continue + } + r.updateMetrics(res) + for name, consumerCh := range r.consumerChannels { + log.Println("sending update, consumer: ", name, "res: ", fmt.Sprintf("%+v", res)) + *consumerCh <- res + } + // close startupComplete if not already closed + select { + case <-startupComplete: + default: + close(startupComplete) + } + } + } + }(ctx) + // trigger initial cluster info call + log.Println("triggering initial cluster info call") + r.sync <- struct{}{} + + // start a ticker routine + go func(ctx context.Context) { + if r.interval <= 0 { + log.Println("no periodic cluster info label update requested") + return + } + ticker := time.NewTicker(r.interval) + for { + select { + case <-ctx.Done(): + log.Println("context cancelled, exiting cluster info trigger loop, err: ", ctx.Err()) + return + case <-ticker.C: + log.Println("triggering periodic update") + r.sync <- struct{}{} + } + } + }(ctx) + + // block until the first retrieval was successful + select { + case <-startupComplete: + // first sync has been successful + log.Println("initial clusterinfo sync succeeded") + return nil + case <-time.After(initialTimeout): + // initial call timed out + return ErrInitialCallTimeout + case <-ctx.Done(): + // context cancelled + return nil + } +} + +func (r *Retriever) fetchAndDecodeClusterInfo() (*Response, error) { + var response *Response + u := *r.url + u.Path = path.Join(r.url.Path, "/") + + res, err := r.client.Get(u.String()) + if err != nil { + log.Println("failed to get cluster info, err: ", err) + return nil, err + } + + defer func() { + err = res.Body.Close() + if err != nil { + log.Println("failed to close http.Client, err: ", err) + } + }() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + bts, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(bts, &response); err != nil { + return nil, err + } + + return response, nil +} diff --git a/inputs/elasticsearch/pkg/clusterinfo/clusterinfo_response.go b/inputs/elasticsearch/pkg/clusterinfo/clusterinfo_response.go new file mode 100644 index 00000000..5e384f0c --- /dev/null +++ b/inputs/elasticsearch/pkg/clusterinfo/clusterinfo_response.go @@ -0,0 +1,36 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterinfo + +import ( + "github.com/blang/semver/v4" +) + +// Response is the cluster info retrievable from the / endpoint +type Response struct { + Name string `json:"name"` + ClusterName string `json:"cluster_name"` + ClusterUUID string `json:"cluster_uuid"` + Version VersionInfo `json:"version"` + Tagline string `json:"tagline"` +} + +// VersionInfo is the version info retrievable from the / endpoint, embedded in Response +type VersionInfo struct { + Number semver.Version `json:"number"` + BuildHash string `json:"build_hash"` + BuildDate string `json:"build_date"` + BuildSnapshot bool `json:"build_snapshot"` + LuceneVersion semver.Version `json:"lucene_version"` +} diff --git a/inputs/elasticsearch/pkg/clusterinfo/clusterinfo_test.go b/inputs/elasticsearch/pkg/clusterinfo/clusterinfo_test.go new file mode 100644 index 00000000..11394983 --- /dev/null +++ b/inputs/elasticsearch/pkg/clusterinfo/clusterinfo_test.go @@ -0,0 +1,219 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterinfo + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "sync" + "testing" + "time" + + "github.com/blang/semver/v4" +) + +const ( + nodeName = "test-node-" + clusterName = "test-cluster-1" + clusterUUID = "r1bT9sBrR7S9-CamE41Qqg" + versionNumber = "5.6.9" + buildHash = "877a590" + buildDate = "2018-04-12T16:25:14.838Z" + buildSnapshot = false + luceneVersion = "6.6.1" + tagline = "You Know, for Search" +) + +type mockES struct{} + +func (mockES) ServeHTTP(w http.ResponseWriter, _ *http.Request) { + + fmt.Fprintf(w, `{ + "name" : "%s", + "cluster_name" : "%s", + "cluster_uuid" : "%s", + "version" : { + "number" : "%s", + "build_hash" : "%s", + "build_date" : "%s", + "build_snapshot" : %t, + "lucene_version" : "%s" + }, + "tagline" : "%s" +}`, + nodeName, + clusterName, + clusterUUID, + versionNumber, + buildHash, + buildDate, + buildSnapshot, + luceneVersion, + tagline, + ) +} + +type mockConsumer struct { + name string + + mu sync.RWMutex + data *Response + + ch chan *Response +} + +func newMockConsumer(ctx context.Context, name string, t *testing.T) *mockConsumer { + mc := &mockConsumer{ + name: name, + ch: make(chan *Response), + } + go func() { + for { + select { + case d := <-mc.ch: + mc.mu.Lock() + mc.data = d + t.Logf("consumer %s received data from channel: %+v\n", mc, mc.data) + mc.mu.Unlock() + case <-ctx.Done(): + return + } + } + }() + return mc +} + +func (mc *mockConsumer) getData() Response { + mc.mu.RLock() + defer mc.mu.RUnlock() + return *mc.data +} + +func (mc *mockConsumer) String() string { + return mc.name +} + +func (mc *mockConsumer) ClusterLabelUpdates() *chan *Response { + return &mc.ch +} + +func TestNew(t *testing.T) { + u, err := url.Parse("http://localhost:9200") + if err != nil { + t.Skipf("internal test error: %s", err) + } + r := New(http.DefaultClient, u, 0) + if r.url != u { + t.Errorf("new Retriever mal-constructed") + } +} + +func TestRetriever_RegisterConsumer(t *testing.T) { + mockES := httptest.NewServer(mockES{}) + u, err := url.Parse(mockES.URL) + if err != nil { + t.Fatalf("internal test error: %s", err) + } + retriever := New(mockES.Client(), u, 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + consumerNames := []string{"consumer-1", "consumer-2"} + for _, n := range consumerNames { + c := newMockConsumer(ctx, n, t) + if err := retriever.RegisterConsumer(c); err != nil { + t.Errorf("failed to register consumer: %s", err) + } + } + if len(retriever.consumerChannels) != len(consumerNames) { + t.Error("number of registered consumerChannels doesn't match the number of calls to the register func") + } +} + +func TestRetriever_fetchAndDecodeClusterInfo(t *testing.T) { + // these override test package globals + versionNumber, _ := semver.Make(versionNumber) + luceneVersion, _ := semver.Make(luceneVersion) + + var expected = &Response{ + Name: nodeName, + ClusterName: clusterName, + ClusterUUID: clusterUUID, + Version: VersionInfo{ + Number: versionNumber, + BuildHash: buildHash, + BuildDate: buildDate, + BuildSnapshot: buildSnapshot, + LuceneVersion: luceneVersion, + }, + Tagline: tagline, + } + + mockES := httptest.NewServer(mockES{}) + u, err := url.Parse(mockES.URL) + if err != nil { + t.Skipf("internal test error: %s", err) + } + retriever := New(mockES.Client(), u, 0) + ci, err := retriever.fetchAndDecodeClusterInfo() + if err != nil { + t.Fatalf("failed to retrieve cluster info: %s", err) + } + + if !reflect.DeepEqual(ci, expected) { + t.Errorf("unexpected response, want %v, got %v", expected, ci) + } +} + +func TestRetriever_Run(t *testing.T) { + // setup mock ES + mockES := httptest.NewServer(mockES{}) + u, err := url.Parse(mockES.URL) + if err != nil { + t.Fatalf("internal test error: %s", err) + } + + // setup cluster info retriever + retriever := New(mockES.Client(), u, 0) + + // setup mock consumer + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + mc := newMockConsumer(ctx, "test-consumer", t) + if err := retriever.RegisterConsumer(mc); err != nil { + t.Fatalf("failed to register consumer: %s", err) + } + + // start retriever + retriever.Run(ctx) + + // trigger update + retriever.Update() + time.Sleep(20 * time.Millisecond) + // ToDo: check mockConsumers received data + t.Logf("%+v\n", mc.getData()) + + // check for deadlocks + select { + case <-ctx.Done(): + if err := ctx.Err(); errors.Is(err, context.DeadlineExceeded) { + t.Fatal("context timeout exceeded, caught deadlock") + } + default: + } +} diff --git a/inputs/elasticsearch/pkg/roundtripper/roundtripper.go b/inputs/elasticsearch/pkg/roundtripper/roundtripper.go new file mode 100644 index 00000000..589c9c32 --- /dev/null +++ b/inputs/elasticsearch/pkg/roundtripper/roundtripper.go @@ -0,0 +1,107 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roundtripper + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "log" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +const ( + service = "es" +) + +type AWSSigningTransport struct { + t http.RoundTripper + creds aws.CredentialsProvider + region string +} + +func NewAWSSigningTransport(transport http.RoundTripper, region string, roleArn string) (*AWSSigningTransport, error) { + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(region)) + if err != nil { + log.Println("failed to load aws default config, err: ", err) + return nil, err + } + + if roleArn != "" { + cfg.Credentials = stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), roleArn) + } + + creds := aws.NewCredentialsCache(cfg.Credentials) + // Run a single fetch credentials operation to ensure that the credentials + // are valid before returning the transport. + _, err = cfg.Credentials.Retrieve(context.Background()) + if err != nil { + log.Println("failed to retrive aws credentials, err: ", err) + return nil, err + } + + return &AWSSigningTransport{ + t: transport, + region: region, + creds: creds, + }, err +} + +func (a *AWSSigningTransport) RoundTrip(req *http.Request) (*http.Response, error) { + signer := v4.NewSigner() + payloadHash, newReader, err := hashPayload(req.Body) + if err != nil { + log.Println("failed to hash request body, err: ", err) + return nil, err + } + req.Body = newReader + + creds, err := a.creds.Retrieve(context.Background()) + if err != nil { + log.Println("failed to retrieve aws credentials, err: ", err) + return nil, err + } + + err = signer.SignHTTP(context.Background(), creds, req, payloadHash, service, a.region, time.Now()) + if err != nil { + log.Println("failed to sign request body, err: ", err) + return nil, err + } + return a.t.RoundTrip(req) +} + +func hashPayload(r io.ReadCloser) (string, io.ReadCloser, error) { + var newReader io.ReadCloser + payload := []byte("") + if r != nil { + defer r.Close() + payload, err := io.ReadAll(r) + if err != nil { + return "", newReader, err + } + newReader = io.NopCloser(bytes.NewReader(payload)) + } + hash := sha256.Sum256(payload) + payloadHash := hex.EncodeToString(hash[:]) + return payloadHash, newReader, nil +}