From 1bcace53434d0df408d11576f244a098390dc653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacio=20Mart=C3=ADnez?= Date: Sun, 21 Jul 2024 22:08:16 +0200 Subject: [PATCH] Add support for Prometheus metrics --- doc/http.md | 6 ++++ go.mod | 7 +++++ go.sum | 6 ++++ pkg/config/config.go | 25 +++++++++-------- pkg/http/api.go | 8 ++++++ pkg/throttle/throttler.go | 58 +++++++++++++++++++++++++++++++++++++++ vendor/modules.txt | 25 +++++++++++++++++ 7 files changed, 124 insertions(+), 11 deletions(-) diff --git a/doc/http.md b/doc/http.md index d060ca0..43930a6 100644 --- a/doc/http.md +++ b/doc/http.md @@ -107,6 +107,12 @@ Notes: - `/recover-host/`: recover a previously skipped host. - `/skipped-hosts`: list currently skipped hosts +### Metric requests + +- `/aggregated-metrics`: aggregated metrics for MySQL clusters in json format. + +- `/metrics`: metrics in Prometheus format. Prefix for these metrics can be modified using `PrometheusNamespace` and `PrometheusSubsystem` in the config. + ### Other requests - `/help`: show all supported request paths diff --git a/go.mod b/go.mod index e79e770..45733c7 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,15 @@ require ( vitess.io/vitess v2.1.1+incompatible ) +require github.com/prometheus/client_golang v0.9.2 + require ( + github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/golang/protobuf v1.2.0 // indirect github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect + github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 // indirect + github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a // indirect ) diff --git a/go.sum b/go.sum index 5f1bb3f..b8e5227 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,7 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= @@ -27,6 +28,7 @@ github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCO github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502 h1:oS2s2j8GP70jE0IDMyA4BO4HgvkNcjHfR40h2fyhc7s= github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502/go.mod h1:JDhu//MMvcPVPH889Xr7DyamEbTLumgDBALGUyXrz1g= @@ -36,9 +38,13 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/pkg/config/config.go b/pkg/config/config.go index 8a54fff..d6e5aa1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -109,22 +109,25 @@ type ConfigurationSettings struct { BackendMySQLCollation string // if specified, use this collation instead of charset when connecting to MySQL backend MemcacheServers []string // if given, freno will report to aggregated values to given memcache MemcachePath string // use as prefix to metric path in memcache key, e.g. if `MemcachePath` is "myprefix" the key would be "myprefix/mysql/maincluster". Default: "freno" - EnableProfiling bool // enable pprof profiling http api + PrometheusNamespace string + PrometheusSubsystem string + EnableProfiling bool // enable pprof profiling http api Stores StoresSettings } func newConfigurationSettings() *ConfigurationSettings { return &ConfigurationSettings{ - ListenPort: 8087, - RaftBind: "127.0.0.1:10008", - RaftDataDir: "", - DefaultRaftPort: 0, - RaftNodes: []string{}, - BackendMySQLHost: "", - BackendMySQLSchema: "", - BackendMySQLPort: 3306, - MemcacheServers: []string{}, - MemcachePath: "freno", + ListenPort: 8087, + RaftBind: "127.0.0.1:10008", + RaftDataDir: "", + DefaultRaftPort: 0, + RaftNodes: []string{}, + BackendMySQLHost: "", + BackendMySQLSchema: "", + BackendMySQLPort: 3306, + MemcacheServers: []string{}, + MemcachePath: "freno", + PrometheusNamespace: "freno", //Debug: false, //ListenSocket: "", //AnExampleListOfStrings: []string{"*"}, diff --git a/pkg/http/api.go b/pkg/http/api.go index 39b4c4c..7d86fe8 100644 --- a/pkg/http/api.go +++ b/pkg/http/api.go @@ -17,6 +17,7 @@ import ( "github.com/rcrowley/go-metrics/exp" "github.com/julienschmidt/httprouter" + "github.com/prometheus/client_golang/prometheus/promhttp" ) // API exposes the contract for the throttler's web API @@ -33,6 +34,7 @@ type API interface { ReadCheckIfExists(w http.ResponseWriter, r *http.Request, _ httprouter.Params) AggregatedMetrics(w http.ResponseWriter, r *http.Request, _ httprouter.Params) MetricsHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params) + PrometheusMetrics(w http.ResponseWriter, r *http.Request, ps httprouter.Params) ThrottleApp(w http.ResponseWriter, r *http.Request, _ httprouter.Params) UnthrottleApp(w http.ResponseWriter, r *http.Request, _ httprouter.Params) ThrottledApps(w http.ResponseWriter, r *http.Request, _ httprouter.Params) @@ -236,6 +238,11 @@ func (api *APIImpl) MetricsHealth(w http.ResponseWriter, r *http.Request, ps htt json.NewEncoder(w).Encode(metricsHealth) } +func (api *APIImpl) PrometheusMetrics(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + handler := promhttp.Handler() + handler.ServeHTTP(w, r) +} + // ThrottleApp forcibly marks given app as throttled. Future requests by this app may be denied. func (api *APIImpl) ThrottleApp(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { storeName := r.URL.Query().Get("store_name") @@ -416,6 +423,7 @@ func ConfigureRoutes(api API) *httprouter.Router { register(router, "/aggregated-metrics", api.AggregatedMetrics) register(router, "/metrics-health", api.MetricsHealth) + register(router, "/metrics", api.PrometheusMetrics) register(router, "/throttle-app/:app", api.ThrottleApp) register(router, "/throttle-app/:app/ratio/:ratio", api.ThrottleApp) diff --git a/pkg/throttle/throttler.go b/pkg/throttle/throttler.go index 90d2b75..2a87a30 100644 --- a/pkg/throttle/throttler.go +++ b/pkg/throttle/throttler.go @@ -3,6 +3,7 @@ package throttle import ( "encoding/json" "fmt" + "github.com/prometheus/client_golang/prometheus" "io/ioutil" "math/rand" "net/http" @@ -49,6 +50,12 @@ func init() { rand.Seed(time.Now().UnixNano()) } +type PrometheusMetrics struct { + value *prometheus.GaugeVec + timestamp *prometheus.GaugeVec + throttleThreshold *prometheus.GaugeVec +} + type Throttler struct { isLeader bool isLeaderFunc func() bool @@ -69,6 +76,8 @@ type Throttler struct { metricsHealth *cache.Cache shareDomainMetricHealth *cache.Cache + prometheusMetrics *PrometheusMetrics + memcacheClient *memcache.Client memcachePath string @@ -81,6 +90,36 @@ type Throttler struct { httpClient *http.Client } +func NewPrometheusMetrics(namespace string, subsystem string) *PrometheusMetrics { + promMetrics := &PrometheusMetrics{ + value: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "value", + Help: "Result value for the cluster", + }, + []string{"store_type", "store_name"}), + timestamp: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "last_update_timestamp", + Help: "Timestamp of the last update", + }, + []string{"store_type", "store_name"}), + throttleThreshold: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "throttle_threshold", + Help: "Throtthe Threshold", + }, []string{"store_type", "store_name"}), + } + prometheus.MustRegister(promMetrics.value) + prometheus.MustRegister(promMetrics.timestamp) + prometheus.MustRegister(promMetrics.throttleThreshold) + + return promMetrics +} + func NewThrottler() *Throttler { throttler := &Throttler{ isLeader: false, @@ -114,6 +153,11 @@ func NewThrottler() *Throttler { throttler.proxysqlClient = proxysql.NewClient(mysqlRefreshInterval) } + throttler.prometheusMetrics = NewPrometheusMetrics( + config.Settings().PrometheusNamespace, + config.Settings().PrometheusSubsystem, + ) + return throttler } @@ -313,6 +357,7 @@ func (throttler *Throttler) refreshMySQLInventory() error { // is immutable and can only be _replaced_. Hence, it's safe to read in a goroutine: go func() error { throttler.mysqlClusterThresholds.Set(clusterName, clusterSettings.ThrottleThreshold, cache.DefaultExpiration) + throttler.prometheusMetrics.throttleThreshold.WithLabelValues("mysql", clusterName).Set(clusterSettings.ThrottleThreshold) if !clusterSettings.HAProxySettings.IsEmpty() { poolName := clusterSettings.HAProxySettings.PoolName totalHosts := []string{} @@ -443,6 +488,8 @@ func (throttler *Throttler) aggregateMySQLMetrics() error { ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount[clusterName] ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold[clusterName] aggregatedMetric := aggregateMySQLProbes(probes, clusterName, throttler.mysqlInventory.InstanceKeyMetrics, throttler.mysqlInventory.ClusterInstanceHttpChecks, ignoreHostsCount, config.Settings().Stores.MySQL.IgnoreDialTcpErrors, ignoreHostsThreshold) + updateMySQLMetrics(throttler.prometheusMetrics, clusterName, aggregatedMetric) + go throttler.aggregatedMetrics.Set(metricName, aggregatedMetric, cache.DefaultExpiration) if throttler.memcacheClient != nil { go func() { @@ -461,6 +508,17 @@ func (throttler *Throttler) aggregateMySQLMetrics() error { return nil } +func updateMySQLMetrics(prometheusMetrics *PrometheusMetrics, clusterName string, metric base.MetricResult) { + const storeType = "mysql" + prometheusMetrics.timestamp.WithLabelValues(storeType, clusterName).Set(float64(time.Now().Unix())) + value, err := metric.Get() + if err != nil { + prometheusMetrics.value.DeleteLabelValues(storeType, clusterName) + } else { + prometheusMetrics.value.WithLabelValues(storeType, clusterName).Set(value) + } +} + func (throttler *Throttler) pushStatusToExpVar() { metrics.DefaultRegistry.Each(func(metricName string, _ interface{}) { if strings.HasPrefix(metricName, "throttled_states.") { diff --git a/vendor/modules.txt b/vendor/modules.txt index 678071f..e083a32 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,6 +4,9 @@ github.com/DATA-DOG/go-sqlmock # github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 ## explicit; go 1.12 github.com/armon/go-metrics +# github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 +## explicit +github.com/beorn7/perks/quantile # github.com/boltdb/bolt v1.3.1 ## explicit github.com/boltdb/bolt @@ -28,6 +31,9 @@ github.com/hashicorp/golang-lru/simplelru # github.com/julienschmidt/httprouter v1.3.0 ## explicit; go 1.7 github.com/julienschmidt/httprouter +# github.com/matttproud/golang_protobuf_extensions v1.0.1 +## explicit +github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502 ## explicit github.com/outbrain/golib/log @@ -36,6 +42,25 @@ github.com/outbrain/golib/tests # github.com/patrickmn/go-cache v2.1.0+incompatible ## explicit github.com/patrickmn/go-cache +# github.com/prometheus/client_golang v0.9.2 +## explicit +github.com/prometheus/client_golang/prometheus +github.com/prometheus/client_golang/prometheus/internal +github.com/prometheus/client_golang/prometheus/promhttp +# github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 +## explicit +github.com/prometheus/client_model/go +# github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 +## explicit +github.com/prometheus/common/expfmt +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg +github.com/prometheus/common/model +# github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a +## explicit +github.com/prometheus/procfs +github.com/prometheus/procfs/internal/util +github.com/prometheus/procfs/nfs +github.com/prometheus/procfs/xfs # github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 ## explicit github.com/rcrowley/go-metrics