Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Add core etcd metrics #317

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[WIP] Add core etcd metrics #317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,17 @@ func run(c *cli.Context) error {
}
ctx := signals.SetupSignalContext()

metricsConfig.ServerTLSConfig = config.ServerTLSConfig
go metrics.Serve(ctx, metricsConfig)
config.MetricsRegisterer = metrics.Registry
_, err := endpoint.Listen(ctx, config)
ep, err := endpoint.Listen(ctx, config)
if err != nil {
return err
}
if ep.Driver == endpoint.ETCDBackend {
return fmt.Errorf("Endpoint driver %s cannot be used in standalone mode", ep.Driver)
}

metricsConfig.ServerTLSConfig = config.ServerTLSConfig
go metrics.Serve(ctx, metricsConfig)

<-ctx.Done()
return ctx.Err()
}
2 changes: 1 addition & 1 deletion pkg/drivers/dqlite/dqlite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ outer:
return nil
}

func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
opts, err := parseOpts(datasourceName)
if err != nil {
return nil, err
Expand Down
3 changes: 1 addition & 2 deletions pkg/drivers/dqlite/no_dqlite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@ import (

"github.com/k3s-io/kine/pkg/drivers/generic"
"github.com/k3s-io/kine/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)

func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
return nil, errors.New(`this binary is built without dqlite support, compile with "-tags dqlite"`)
}
7 changes: 2 additions & 5 deletions pkg/drivers/generic/generic.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/k3s-io/kine/pkg/metrics"
"github.com/k3s-io/kine/pkg/server"
"github.com/k3s-io/kine/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/sirupsen/logrus"
)
Expand Down Expand Up @@ -171,7 +170,7 @@ func openAndTest(driverName, dataSourceName string) (*sql.DB, error) {
return db, nil
}

func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool, metricsRegisterer prometheus.Registerer) (*Generic, error) {
func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool) (*Generic, error) {
var (
db *sql.DB
err error
Expand All @@ -193,9 +192,7 @@ func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig

configureConnectionPooling(connPoolConfig, db, driverName)

if metricsRegisterer != nil {
metricsRegisterer.MustRegister(collectors.NewDBStatsCollector(db, "kine"))
}
metrics.DefaultRegisterer.MustRegister(collectors.NewDBStatsCollector(db, "kine"))

return &Generic{
DB: db,
Expand Down
5 changes: 2 additions & 3 deletions pkg/drivers/mysql/mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"strconv"

"github.com/go-sql-driver/mysql"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"

"github.com/k3s-io/kine/pkg/drivers/generic"
Expand Down Expand Up @@ -52,7 +51,7 @@ var (
createDB = "CREATE DATABASE IF NOT EXISTS "
)

func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
tlsConfig, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
Expand All @@ -71,7 +70,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
return nil, err
}

dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false, metricsRegisterer)
dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false)
if err != nil {
return nil, err
}
Expand Down
5 changes: 2 additions & 3 deletions pkg/drivers/pgsql/pgsql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"github.com/k3s-io/kine/pkg/server"
"github.com/k3s-io/kine/pkg/tls"
"github.com/k3s-io/kine/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)

Expand Down Expand Up @@ -53,7 +52,7 @@ var (
createDB = "CREATE DATABASE "
)

func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
parsedDSN, err := prepareDSN(dataSourceName, tlsInfo)
if err != nil {
return nil, err
Expand All @@ -63,7 +62,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
return nil, err
}

dialect, err := generic.Open(ctx, "pgx", parsedDSN, connPoolConfig, "$", true, metricsRegisterer)
dialect, err := generic.Open(ctx, "pgx", parsedDSN, connPoolConfig, "$", true)
if err != nil {
return nil, err
}
Expand Down
9 changes: 4 additions & 5 deletions pkg/drivers/sqlite/sqlite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/k3s-io/kine/pkg/util"
"github.com/mattn/go-sqlite3"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"

// sqlite db driver
Expand Down Expand Up @@ -47,20 +46,20 @@ var (
}
)

func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig, metricsRegisterer)
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig)
return backend, err
}

func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, *generic.Generic, error) {
func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, *generic.Generic, error) {
if dataSourceName == "" {
if err := os.MkdirAll("./db", 0700); err != nil {
return nil, nil, err
}
dataSourceName = "./db/state.db?_journal=WAL&cache=shared&_busy_timeout=30000"
}

dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false, metricsRegisterer)
dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false)
if err != nil {
return nil, nil, err
}
Expand Down
42 changes: 16 additions & 26 deletions pkg/endpoint/endpoint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@ import (
"github.com/k3s-io/kine/pkg/drivers/nats"
"github.com/k3s-io/kine/pkg/drivers/pgsql"
"github.com/k3s-io/kine/pkg/drivers/sqlite"
"github.com/k3s-io/kine/pkg/metrics"
"github.com/k3s-io/kine/pkg/server"
"github.com/k3s-io/kine/pkg/tls"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
"go.etcd.io/etcd/server/v3/embed"
"google.golang.org/grpc"
Expand All @@ -44,21 +42,22 @@ type Config struct {
ConnectionPoolConfig generic.ConnectionPoolConfig
ServerTLSConfig tls.Config
BackendTLSConfig tls.Config
MetricsRegisterer prometheus.Registerer
NotifyInterval time.Duration
EmulatedETCDVersion string
}

type ETCDConfig struct {
Driver string
Endpoints []string
TLSConfig tls.Config
LeaderElect bool
}

func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
func Listen(ctx context.Context, config Config) (*ETCDConfig, error) {
driver, dsn := ParseStorageEndpoint(config.Endpoint)
if driver == ETCDBackend {
return ETCDConfig{
return &ETCDConfig{
Driver: driver,
Endpoints: strings.Split(config.Endpoint, ","),
TLSConfig: config.BackendTLSConfig,
LeaderElect: true,
Expand All @@ -67,33 +66,25 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) {

leaderelect, backend, err := getKineStorageBackend(ctx, driver, dsn, config)
if err != nil {
return ETCDConfig{}, errors.Wrap(err, "building kine")
}

if config.MetricsRegisterer != nil {
config.MetricsRegisterer.MustRegister(
metrics.SQLTotal,
metrics.SQLTime,
metrics.CompactTotal,
)
return nil, errors.Wrap(err, "building kine")
}

if err := backend.Start(ctx); err != nil {
return ETCDConfig{}, errors.Wrap(err, "starting kine backend")
return nil, errors.Wrap(err, "starting kine backend")
}

// set up GRPC server and register services
b := server.New(backend, endpointScheme(config), config.NotifyInterval, config.EmulatedETCDVersion)
grpcServer, err := grpcServer(config)
if err != nil {
return ETCDConfig{}, errors.Wrap(err, "creating GRPC server")
return nil, errors.Wrap(err, "creating GRPC server")
}
b.Register(grpcServer)

// Create raw listener and wrap in cmux for protocol switching
listener, err := createListener(config)
if err != nil {
return ETCDConfig{}, errors.Wrap(err, "creating listener")
return nil, errors.Wrap(err, "creating listener")
}

go func() {
Expand All @@ -105,12 +96,11 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
endpoint := endpointURL(config, listener)
logrus.Infof("Kine available at %s", endpoint)

return ETCDConfig{
LeaderElect: leaderelect,
return &ETCDConfig{
Driver: driver,
Endpoints: []string{endpoint},
TLSConfig: tls.Config{
CAFile: config.ServerTLSConfig.CAFile,
},
TLSConfig: tls.Config{CAFile: config.ServerTLSConfig.CAFile},
LeaderElect: leaderelect,
}, nil
}

Expand Down Expand Up @@ -214,13 +204,13 @@ func getKineStorageBackend(ctx context.Context, driver, dsn string, cfg Config)
switch driver {
case SQLiteBackend:
leaderElect = false
backend, err = sqlite.New(ctx, dsn, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = sqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
case DQLiteBackend:
backend, err = dqlite.New(ctx, dsn, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = dqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
case PostgresBackend:
backend, err = pgsql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = pgsql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig)
case MySQLBackend:
backend, err = mysql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
backend, err = mysql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig)
case JetStreamBackend:
backend, err = nats.NewLegacy(ctx, dsn, cfg.BackendTLSConfig)
case NATSBackend:
Expand Down
Loading