Skip to content

Commit

Permalink
Add core etcd metrics
Browse files Browse the repository at this point in the history 
* Refuse to run standalone in passthrough mode.
  It didn't actually do anything anyway.
* Simplify metrics.
  Switch to standard DefaultRegisterer/DefaultGatherer instead of passing around a Registry.
* Add etcd metrics from upstream.
  Copied from mvcc and server; they are not exported so we cannot reuse them.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
  • Loading branch information
@brandond
brandond committed Aug 22, 2024
1 parent 227f4d3 commit eb9ac0e
Show file tree
Hide file tree
Showing 11 changed files with 321 additions and 68 deletions.
12 changes: 8 additions & 4 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,17 @@ func run(c *cli.Context) error {
}
ctx := signals.SetupSignalContext()

metricsConfig.ServerTLSConfig = config.ServerTLSConfig
go metrics.Serve(ctx, metricsConfig)
config.MetricsRegisterer = metrics.Registry
_, err := endpoint.Listen(ctx, config)
ep, err := endpoint.Listen(ctx, config)
if err != nil {
return err
}
if ep.Driver == endpoint.ETCDBackend {
return fmt.Errorf("Endpoint driver %s cannot be used in standalone mode", ep.Driver)
}

metricsConfig.ServerTLSConfig = config.ServerTLSConfig
go metrics.Serve(ctx, metricsConfig)

<-ctx.Done()
return ctx.Err()
}
2 changes: 1 addition & 1 deletion pkg/drivers/dqlite/dqlite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ outer:
return nil
}

func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
opts, err := parseOpts(datasourceName)
if err != nil {
return nil, err
Expand Down
3 changes: 1 addition & 2 deletions pkg/drivers/dqlite/no_dqlite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@ import (

"github.com/k3s-io/kine/pkg/drivers/generic"
"github.com/k3s-io/kine/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)

func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
return nil, errors.New(`this binary is built without dqlite support, compile with "-tags dqlite"`)
}
7 changes: 2 additions & 5 deletions pkg/drivers/generic/generic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/k3s-io/kine/pkg/metrics"
"github.com/k3s-io/kine/pkg/server"
"github.com/k3s-io/kine/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/sirupsen/logrus"
)
Expand Down Expand Up @@ -171,7 +170,7 @@ func openAndTest(driverName, dataSourceName string) (*sql.DB, error) {
return db, nil
}

func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool, metricsRegisterer prometheus.Registerer) (*Generic, error) {
func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool) (*Generic, error) {
var (
db *sql.DB
err error
Expand All @@ -193,9 +192,7 @@ func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig

configureConnectionPooling(connPoolConfig, db, driverName)

if metricsRegisterer != nil {
metricsRegisterer.MustRegister(collectors.NewDBStatsCollector(db, "kine"))
}
metrics.DefaultRegisterer.MustRegister(collectors.NewDBStatsCollector(db, "kine"))

return &Generic{
DB: db,
Expand Down
5 changes: 2 additions & 3 deletions pkg/drivers/mysql/mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"strconv"

"github.com/go-sql-driver/mysql"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"

"github.com/k3s-io/kine/pkg/drivers/generic"
Expand Down Expand Up @@ -52,7 +51,7 @@ var (
createDB = "CREATE DATABASE IF NOT EXISTS "
)

func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
tlsConfig, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
Expand All @@ -71,7 +70,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
return nil, err
}

dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false, metricsRegisterer)
dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false)
if err != nil {
return nil, err
}
Expand Down
5 changes: 2 additions & 3 deletions pkg/drivers/pgsql/pgsql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"github.com/k3s-io/kine/pkg/server"
"github.com/k3s-io/kine/pkg/tls"
"github.com/k3s-io/kine/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)

Expand Down Expand Up @@ -53,7 +52,7 @@ var (
createDB = "CREATE DATABASE "
)

func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
parsedDSN, err := prepareDSN(dataSourceName, tlsInfo)
if err != nil {
return nil, err
Expand All @@ -63,7 +62,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
return nil, err
}

dialect, err := generic.Open(ctx, "pgx", parsedDSN, connPoolConfig, "$", true, metricsRegisterer)
dialect, err := generic.Open(ctx, "pgx", parsedDSN, connPoolConfig, "$", true)
if err != nil {
return nil, err
}
Expand Down
9 changes: 4 additions & 5 deletions pkg/drivers/sqlite/sqlite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/k3s-io/kine/pkg/util"
"github.com/mattn/go-sqlite3"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"

// sqlite db driver
Expand Down Expand Up @@ -47,20 +46,20 @@ var (
}
)

func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig, metricsRegisterer)
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig)
return backend, err
}

func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, *generic.Generic, error) {
func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, *generic.Generic, error) {
if dataSourceName == "" {
if err := os.MkdirAll("./db", 0700); err != nil {
return nil, nil, err
}
dataSourceName = "./db/state.db?_journal=WAL&cache=shared&_busy_timeout=30000"
}

dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false, metricsRegisterer)
dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false)
if err != nil {
return nil, nil, err
}
Expand Down
42 changes: 16 additions & 26 deletions pkg/endpoint/endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@ import (
"github.com/k3s-io/kine/pkg/drivers/nats"
"github.com/k3s-io/kine/pkg/drivers/pgsql"
"github.com/k3s-io/kine/pkg/drivers/sqlite"
"github.com/k3s-io/kine/pkg/metrics"
"github.com/k3s-io/kine/pkg/server"
"github.com/k3s-io/kine/pkg/tls"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
"go.etcd.io/etcd/server/v3/embed"
"google.golang.org/grpc"
Expand All @@ -44,21 +42,22 @@ type Config struct {
ConnectionPoolConfig generic.ConnectionPoolConfig
ServerTLSConfig tls.Config
BackendTLSConfig tls.Config
MetricsRegisterer prometheus.Registerer
NotifyInterval time.Duration
EmulatedETCDVersion string
}

type ETCDConfig struct {
Driver string
Endpoints []string
TLSConfig tls.Config
LeaderElect bool
}

func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
func Listen(ctx context.Context, config Config) (*ETCDConfig, error) {
driver, dsn := ParseStorageEndpoint(config.Endpoint)
if driver == ETCDBackend {
return ETCDConfig{
return &ETCDConfig{
Driver: driver,
Endpoints: strings.Split(config.Endpoint, ","),
TLSConfig: config.BackendTLSConfig,
LeaderElect: true,
Expand All @@ -67,33 +66,25 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) {

leaderelect, backend, err := getKineStorageBackend(ctx, driver, dsn, config)
if err != nil {
return ETCDConfig{}, errors.Wrap(err, "building kine")
}

if config.MetricsRegisterer != nil {
config.MetricsRegisterer.MustRegister(
metrics.SQLTotal,
metrics.SQLTime,
metrics.CompactTotal,
)
return nil, errors.Wrap(err, "building kine")
}

if err := backend.Start(ctx); err != nil {
return ETCDConfig{}, errors.Wrap(err, "starting kine backend")
return nil, errors.Wrap(err, "starting kine backend")
}

// set up GRPC server and register services
b := server.New(backend, endpointScheme(config), config.NotifyInterval, config.EmulatedETCDVersion)
grpcServer, err := grpcServer(config)
if err != nil {
return ETCDConfig{}, errors.Wrap(err, "creating GRPC server")
return nil, errors.Wrap(err, "creating GRPC server")
}
b.Register(grpcServer)

// Create raw listener and wrap in cmux for protocol switching
listener, err := createListener(config)
if err != nil {
return ETCDConfig{}, errors.Wrap(err, "creating listener")
return nil, errors.Wrap(err, "creating listener")
}

go func() {
Expand All @@ -105,12 +96,11 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
endpoint := endpointURL(config, listener)
logrus.Infof("Kine available at %s", endpoint)

return ETCDConfig{
LeaderElect: leaderelect,
return &ETCDConfig{
Driver: driver,
Endpoints: []string{endpoint},
TLSConfig: tls.Config{
CAFile: config.ServerTLSConfig.CAFile,
},
TLSConfig: tls.Config{CAFile: config.ServerTLSConfig.CAFile},
LeaderElect: leaderelect,
}, nil
}

Expand Down Expand Up @@ -214,13 +204,13 @@ func getKineStorageBackend(ctx context.Context, driver, dsn string, cfg Config)
switch driver {
case SQLiteBackend:
leaderElect = false
backend, err = sqlite.New(ctx, dsn, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = sqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
case DQLiteBackend:
backend, err = dqlite.New(ctx, dsn, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = dqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
case PostgresBackend:
backend, err = pgsql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = pgsql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig)
case MySQLBackend:
backend, err = mysql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = mysql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig)
case JetStreamBackend:
backend, err = nats.NewLegacy(ctx, dsn, cfg.BackendTLSConfig)
case NATSBackend:
Expand Down
Loading

0 comments on commit eb9ac0e

Please sign in to comment.