[PIV-176] Bpfink metrics for logs/events caught (#34)

cmd/main.go

@@ -57,6 +57,11 @@ type (
 		File  string
 		IsDir bool
 	}
+
+	LogHook struct {
+		metric *pkg.Metrics
+		config Configuration
+	}
 )
 
 const (
@@ -68,6 +73,19 @@ const (
 	keySize               = 16
 )
 
+// LogHook to send a graphite metric for each log entry
+func (h LogHook) Run(e *zerolog.Event, level zerolog.Level, msg string) {
+	metrics, err := h.config.metrics()
+	if err != nil {
+		return
+	}
+	h.metric = metrics
+	h.metric.RecordByLogTypes(level.String())
+	if err = h.metric.Init(); err != nil {
+		return
+	}
+}
+
 func (c Configuration) logger() (logger zerolog.Logger) {
 	lvlMap := map[string]zerolog.Level{
 		"debug": zerolog.DebugLevel,
@@ -79,12 +97,12 @@ func (c Configuration) logger() (logger zerolog.Logger) {
 
 	if c.Debug {
 		logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).
-			With().Timestamp().Logger().Level(lvlMap["debug"])
+			With().Timestamp().Logger().Level(lvlMap["debug"]).Hook(LogHook{config: c})
 	} else {
 		// We can't use journald from rsyslog as it is way too complicated to find
 		// a good documentation on both of those projects
 		// logger = zerolog.New(journald.NewJournalDWriter()).Level(lvlMap[c.Level])
-		logger = zerolog.New(os.Stderr).Level(lvlMap[c.Level])
+		logger = zerolog.New(os.Stderr).Level(lvlMap[c.Level]).Hook(LogHook{config: c})
 	}
 	return logger
 }
@@ -273,13 +291,11 @@ func (c Configuration) resolvePath(pathFull string) (string, os.FileInfo) {
 }
 
 func (c Configuration) metrics() (*pkg.Metrics, error) {
-	logger := c.logger()
 	metrics := &pkg.Metrics{
 		GraphiteHost:    c.MetricsConfig.GraphiteHost,
 		Namespace:       c.MetricsConfig.NameSpace,
 		GraphiteMode:    c.MetricsConfig.GraphiteMode,
 		MetricsInterval: c.MetricsConfig.CollectionInterval,
-		Logger:          logger,
 	}
 
 	hostname, err := os.Hostname()
@@ -308,7 +324,7 @@ func (c Configuration) metrics() (*pkg.Metrics, error) {
 			}
 		}
 		if err = file.Close(); err != nil {
-			logger.Error().Err(err)
+			return nil, err
 		}
 	}
 
@@ -392,6 +408,7 @@ func run() error {
 			Err(err).
 			Msgf("failed to init metrics: %v", err)
 	}
+	metrics.Logger = logger
 
 	if viper.GetInt("graphite-mode") != 0 {
 		metrics.GraphiteMode = viper.GetInt("graphite-mode")
@@ -421,6 +438,7 @@ func run() error {
 	if err != nil {
 		return err
 	}
+	watcher.Metrics = metrics
 	if err = metrics.Init(); err != nil {
 		return err
 	}

pkg/graphite.go

@@ -74,6 +74,26 @@ func (m *Metrics) Init() error {
 	return nil
 }
 
+// RecordByLogTypes sends count of different types of logs
+func (m *Metrics) RecordByLogTypes(logType string) {
+	metricNameByHost := fmt.Sprintf("log_level.%s.by_host.%s.count.hourly", logType, quote(m.Hostname))
+	goMetrics.GetOrRegisterGauge(metricNameByHost, m.EveryHourRegister).Update(int64(1))
+	if m.RoleName != "" {
+		metricNameByRole := fmt.Sprintf("log_level.%s.by_role.%s.%s.count.hourly", logType, quote(m.RoleName), quote(m.Hostname))
+		goMetrics.GetOrRegisterGauge(metricNameByRole, m.EveryHourRegister).Update(int64(1))
+	}
+}
+
+// RecordByEventsCaught sends count of number of events caught by ebpf
+func (m *Metrics) RecordByEventsCaught() {
+	metricNameByHost := fmt.Sprintf("bpf.events_caught.by_host.%s.count.hourly", quote(m.Hostname))
+	goMetrics.GetOrRegisterGauge(metricNameByHost, m.EveryHourRegister).Update(int64(1))
+	if m.RoleName != "" {
+		metricNameByRole := fmt.Sprintf("bpf.events_caught.by_role.%s.%s.count.hourly", quote(m.RoleName), quote(m.Hostname))
+		goMetrics.GetOrRegisterGauge(metricNameByRole, m.EveryHourRegister).Update(int64(1))
+	}
+}
+
 // RecordByInstalledHost graphite metric to show how manay host have bpfink installed
 func (m *Metrics) RecordByInstalledHost() {
 	metricNameByHost := fmt.Sprintf("installed.by_host.%s.count.hourly", quote(m.Hostname))

pkg/watcher.go

@@ -22,6 +22,7 @@ type (
 		CloseChannels chan struct{}
 		Excludes      []string
 		Sudoers       []string
+		Metrics       *Metrics
 	}
 	// Register defines register interface for a watcher
 	Register interface {
@@ -232,6 +233,8 @@ func (w *Watcher) Start() error {
 	for {
 		select {
 		case event := <-w.Events:
+			// Send metric to graphite for every event caught, increement by 1
+			w.Metrics.RecordByEventsCaught()
 			switch event.Mode {
 			case dirCreate:
 				w.addInode(&event, true)