From 154a6d12ad9ab95ed8f088e3122e39c9fde0b67a Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 31 Jan 2024 14:11:10 +0100 Subject: [PATCH 01/31] Support custom trackers for received bytes. --- pkg/distributor/limits.go | 3 +++ pkg/loghttp/push/push.go | 44 ++++++++++++++++++++++++++++++++++++++- pkg/validation/limits.go | 4 ++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/pkg/distributor/limits.go b/pkg/distributor/limits.go index 6db6995662dd2..228d870f75feb 100644 --- a/pkg/distributor/limits.go +++ b/pkg/distributor/limits.go @@ -31,4 +31,7 @@ type Limits interface { MaxStructuredMetadataSize(userID string) int MaxStructuredMetadataCount(userID string) int OTLPConfig(userID string) push.OTLPConfig + + // TODO: return the custom tracker config instead. + CustomTrackerMatchers(userID string) []string } diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 15b7bba0a78c9..b078b58454058 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -36,6 +36,11 @@ var ( Name: "distributor_bytes_received_total", Help: "The total number of uncompressed bytes received per tenant. Includes structured metadata bytes.", }, []string{"tenant", "retention_hours"}) + bytesIngestedCustom = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: constants.Loki, + Name: "distributor_bytes_received_custom_tracker_total", + Help: "The total number of uncompressed bytes received per tenant. Includes structured metadata bytes.", + }, []string{"tenant", "retention_hours", "tacker"}) structuredMetadataBytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_structured_metadata_bytes_received_total", @@ -60,6 +65,7 @@ type TenantsRetention interface { type Limits interface { OTLPConfig(userID string) OTLPConfig + CustomTrackerMatchers(userID string) []string } type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) @@ -74,6 +80,9 @@ type Stats struct { contentType string contentEncoding string bodySize int64 + + logLinesBytesCustomTracker map[string]map[time.Duration]int64 + structuredMetadataBytesCustomTracker map[string]map[time.Duration]int64 } func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser) (*logproto.PushRequest, error) { @@ -97,6 +106,28 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete entriesSize += size } + // Process custom trackers + for name, logLinesBytes := range pushStats.logLinesBytesCustomTracker { + for retentionPeriod, size := range logLinesBytes { + var retentionHours string + if retentionPeriod > 0 { + retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) + } + + bytesIngestedCustom.WithLabelValues(userID, retentionHours, name).Add(float64(size)) + } + } + for name, structuredMetadataBytes := range pushStats.structuredMetadataBytesCustomTracker { + for retentionPeriod, size := range structuredMetadataBytes { + var retentionHours string + if retentionPeriod > 0 { + retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) + } + + bytesIngestedCustom.WithLabelValues(userID, retentionHours, name).Add(float64(size)) + } + } + for retentionPeriod, size := range pushStats.structuredMetadataBytes { var retentionHours string if retentionPeriod > 0 { @@ -135,7 +166,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete return req, nil } -func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, _ Limits) (*logproto.PushRequest, *Stats, error) { +func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) { // Body var body io.Reader // bodySize should always reflect the compressed size of the request body @@ -225,6 +256,17 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe if e.Timestamp.After(pushStats.mostRecentEntryTimestamp) { pushStats.mostRecentEntryTimestamp = e.Timestamp } + + trackers := limits.CustomTrackerMatchers(userID) + for _, t := range trackers { + // if matchers s.Labels + logLinesBytes, ok := pushStats.structuredMetadataBytesCustomTracker[t] + if !ok { + pushStats.structuredMetadataBytesCustomTracker[t] = map[time.Duration]int64{} + logLinesBytes = pushStats.structuredMetadataBytesCustomTracker[t] + } + logLinesBytes[retentionPeriod] += int64(len(e.Line)) + } } } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index e3052c1781b89..9d22f2c3d408d 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -902,6 +902,10 @@ func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig { return o.getOverridesForUser(userID).OTLPConfig } +func (o *Overrides) CustomTrackerMatchers(userID string) []string { + return nil +} + func (o *Overrides) getOverridesForUser(userID string) *Limits { if o.tenantLimits != nil { l := o.tenantLimits.TenantLimits(userID) From 79fff6bd688ee83991fb9785747d37877593c6a9 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 31 Jan 2024 16:21:55 +0100 Subject: [PATCH 02/31] Implement custom trackers config --- pkg/distributor/limits.go | 4 +-- pkg/loghttp/push/custom_trackers.go | 54 ++++++++++++++++++++++++++++ pkg/loghttp/push/push.go | 55 +++++++++++++++-------------- pkg/logql/syntax/parser.go | 1 + pkg/validation/limits.go | 6 ++-- 5 files changed, 88 insertions(+), 32 deletions(-) create mode 100644 pkg/loghttp/push/custom_trackers.go diff --git a/pkg/distributor/limits.go b/pkg/distributor/limits.go index 228d870f75feb..4c3416d1242b9 100644 --- a/pkg/distributor/limits.go +++ b/pkg/distributor/limits.go @@ -31,7 +31,5 @@ type Limits interface { MaxStructuredMetadataSize(userID string) int MaxStructuredMetadataCount(userID string) int OTLPConfig(userID string) push.OTLPConfig - - // TODO: return the custom tracker config instead. - CustomTrackerMatchers(userID string) []string + CustomTrackersConfig(userID string) push.CustomTrackersConfig } diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go new file mode 100644 index 0000000000000..a52756f061b7a --- /dev/null +++ b/pkg/loghttp/push/custom_trackers.go @@ -0,0 +1,54 @@ +package push + +import ( + "fmt" + + "github.com/prometheus/prometheus/model/labels" + + "github.com/grafana/loki/pkg/logql/syntax" +) + +type CustomTrackersConfig struct { + source map[string]string + config map[string][]*labels.Matcher +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +// CustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. +func (c *CustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + stringMap := map[string]string{} + err := unmarshal(&stringMap) + if err != nil { + return err + } + *c, err = NewCustomTrackersConfig(stringMap) + return err +} + +func NewCustomTrackersConfig(m map[string]string) (c CustomTrackersConfig, err error) { + c.source = m + c.config = map[string][]*labels.Matcher{} + for name, selector := range m { + matchers, err := syntax.ParseMatchers(selector, true) + if err != nil { + return c, fmt.Errorf("invalid labels matchers: %w", err) + } + c.config[name] = matchers + } + return c, nil +} + +// MatchTrackers returns a list of names of all trackers that match the given labels. +func (c *CustomTrackersConfig) MatchTrackers(lbs labels.Labels) []string { + trackers := make([]string, 0) +Outer: + for name, matchers := range c.config { + for _, m := range matchers { + if !m.Matches(lbs.Get(m.Name)) { + continue Outer + } + } + trackers = append(trackers, name) + } + return trackers +} diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index b078b58454058..0dcb5baac7a20 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -65,7 +65,7 @@ type TenantsRetention interface { type Limits interface { OTLPConfig(userID string) OTLPConfig - CustomTrackerMatchers(userID string) []string + CustomTrackersConfig(userID string) CustomTrackersConfig } type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) @@ -81,8 +81,8 @@ type Stats struct { contentEncoding string bodySize int64 - logLinesBytesCustomTracker map[string]map[time.Duration]int64 - structuredMetadataBytesCustomTracker map[string]map[time.Duration]int64 + logLinesBytesCustomTrackers map[string]map[time.Duration]int64 + structuredMetadataBytesCustomTrackers map[string]map[time.Duration]int64 } func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser) (*logproto.PushRequest, error) { @@ -96,10 +96,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete structuredMetadataSize int64 ) for retentionPeriod, size := range pushStats.logLinesBytes { - var retentionHours string - if retentionPeriod > 0 { - retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) - } + retentionHours := retentionPeriodToString(retentionPeriod) bytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) bytesReceivedStats.Inc(size) @@ -107,32 +104,23 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete } // Process custom trackers - for name, logLinesBytes := range pushStats.logLinesBytesCustomTracker { + for name, logLinesBytes := range pushStats.logLinesBytesCustomTrackers { for retentionPeriod, size := range logLinesBytes { - var retentionHours string - if retentionPeriod > 0 { - retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) - } + retentionHours := retentionPeriodToString(retentionPeriod) bytesIngestedCustom.WithLabelValues(userID, retentionHours, name).Add(float64(size)) } } - for name, structuredMetadataBytes := range pushStats.structuredMetadataBytesCustomTracker { + for name, structuredMetadataBytes := range pushStats.structuredMetadataBytesCustomTrackers { for retentionPeriod, size := range structuredMetadataBytes { - var retentionHours string - if retentionPeriod > 0 { - retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) - } + retentionHours := retentionPeriodToString(retentionPeriod) bytesIngestedCustom.WithLabelValues(userID, retentionHours, name).Add(float64(size)) } } for retentionPeriod, size := range pushStats.structuredMetadataBytes { - var retentionHours string - if retentionPeriod > 0 { - retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) - } + retentionHours := retentionPeriodToString(retentionPeriod) structuredMetadataBytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) bytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) @@ -257,13 +245,18 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe pushStats.mostRecentEntryTimestamp = e.Timestamp } - trackers := limits.CustomTrackerMatchers(userID) - for _, t := range trackers { - // if matchers s.Labels - logLinesBytes, ok := pushStats.structuredMetadataBytesCustomTracker[t] + trackers := limits.CustomTrackersConfig(userID) + + // TODO: do not duplicate + lbs, err := syntax.ParseLabels(s.Labels) + if err != nil { + return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) + } + for _, t := range trackers.MatchTrackers(lbs) { + logLinesBytes, ok := pushStats.structuredMetadataBytesCustomTrackers[t] if !ok { - pushStats.structuredMetadataBytesCustomTracker[t] = map[time.Duration]int64{} - logLinesBytes = pushStats.structuredMetadataBytesCustomTracker[t] + pushStats.structuredMetadataBytesCustomTrackers[t] = map[time.Duration]int64{} + logLinesBytes = pushStats.structuredMetadataBytesCustomTrackers[t] } logLinesBytes[retentionPeriod] += int64(len(e.Line)) } @@ -272,3 +265,11 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe return &req, pushStats, nil } + +func retentionPeriodToString(retentionPeriod time.Duration) string { + var retentionHours string + if retentionPeriod > 0 { + retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) + } + return retentionHours +} diff --git a/pkg/logql/syntax/parser.go b/pkg/logql/syntax/parser.go index 79213049f376c..bd5ffebc3cdb0 100644 --- a/pkg/logql/syntax/parser.go +++ b/pkg/logql/syntax/parser.go @@ -252,6 +252,7 @@ func ParseLogSelector(input string, validate bool) (LogSelectorExpr, error) { // ParseLabels parses labels from a string using logql parser. func ParseLabels(lbs string) (labels.Labels, error) { + // TODO: I wonder if we should use memoization here. ls, err := promql_parser.ParseMetric(lbs) if err != nil { return nil, err diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 9d22f2c3d408d..e4b9f1607ca5c 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -201,6 +201,8 @@ type Limits struct { MaxStructuredMetadataSize flagext.ByteSize `yaml:"max_structured_metadata_size" json:"max_structured_metadata_size" doc:"description=Maximum size accepted for structured metadata per log line."` MaxStructuredMetadataEntriesCount int `yaml:"max_structured_metadata_entries_count" json:"max_structured_metadata_entries_count" doc:"description=Maximum number of structured metadata entries per log line."` OTLPConfig push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"` + + CustomTrackersConfig push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes."` } type StreamRetention struct { @@ -902,8 +904,8 @@ func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig { return o.getOverridesForUser(userID).OTLPConfig } -func (o *Overrides) CustomTrackerMatchers(userID string) []string { - return nil +func (o *Overrides) CustomTrackersConfig(userID string) push.CustomTrackersConfig { + return o.getOverridesForUser(userID).CustomTrackersConfig } func (o *Overrides) getOverridesForUser(userID string) *Limits { From fd1ff0197e2d93ac054f856a5a0d2e64bcb4c23d Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 1 Feb 2024 12:43:10 +0100 Subject: [PATCH 03/31] Note todos --- pkg/loghttp/push/push.go | 2 ++ pkg/validation/validate.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 0dcb5baac7a20..6c8eb209d862e 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -36,6 +36,8 @@ var ( Name: "distributor_bytes_received_total", Help: "The total number of uncompressed bytes received per tenant. Includes structured metadata bytes.", }, []string{"tenant", "retention_hours"}) + + // TODO: track discarded bytes as well bytesIngestedCustom = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_bytes_received_custom_tracker_total", diff --git a/pkg/validation/validate.go b/pkg/validation/validate.go index 09c444aa64987..8b93a2e906a2e 100644 --- a/pkg/validation/validate.go +++ b/pkg/validation/validate.go @@ -102,6 +102,7 @@ var MutatedBytes = promauto.NewCounterVec( []string{ReasonLabel, "truncated"}, ) +// TODO: add custom tracker // DiscardedBytes is a metric of the total discarded bytes, by reason. var DiscardedBytes = promauto.NewCounterVec( prometheus.CounterOpts{ From a787cf449c74bc180ff4f4af073bf644db2ff5b1 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 1 Feb 2024 15:51:26 +0100 Subject: [PATCH 04/31] Fix test --- pkg/loghttp/push/push_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index fa1e2fb28d115..e495622ec0715 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -200,7 +200,8 @@ func TestParseRequest(t *testing.T) { request.Header.Add("Content-Encoding", test.contentEncoding) } - data, err := ParseRequest(util_log.Logger, "fake", request, nil, nil, ParseLokiRequest) + limits := &mockLimits{} + data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest) structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived previousStructuredMetadataBytesReceived += structuredMetadataBytesReceived @@ -231,3 +232,15 @@ func TestParseRequest(t *testing.T) { }) } } + +type mockLimits struct{} + +// CustomTrackersConfig implements Limits. +func (*mockLimits) CustomTrackersConfig(userID string) CustomTrackersConfig { + return CustomTrackersConfig{} +} + +// OTLPConfig implements Limits. +func (*mockLimits) OTLPConfig(userID string) OTLPConfig { + return DefaultOTLPConfig +} From ac8ae3a6bbc4002a5fa35100c1df378d2f5efc42 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 1 Feb 2024 15:53:48 +0100 Subject: [PATCH 05/31] Move code block --- pkg/loghttp/push/push.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 6c8eb209d862e..37918300c8045 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -105,6 +105,18 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete entriesSize += size } + for retentionPeriod, size := range pushStats.structuredMetadataBytes { + retentionHours := retentionPeriodToString(retentionPeriod) + + structuredMetadataBytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) + bytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) + bytesReceivedStats.Inc(size) + structuredMetadataBytesReceivedStats.Inc(size) + + entriesSize += size + structuredMetadataSize += size + } + // Process custom trackers for name, logLinesBytes := range pushStats.logLinesBytesCustomTrackers { for retentionPeriod, size := range logLinesBytes { @@ -121,18 +133,6 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete } } - for retentionPeriod, size := range pushStats.structuredMetadataBytes { - retentionHours := retentionPeriodToString(retentionPeriod) - - structuredMetadataBytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) - bytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) - bytesReceivedStats.Inc(size) - structuredMetadataBytesReceivedStats.Inc(size) - - entriesSize += size - structuredMetadataSize += size - } - // incrementing tenant metrics if we have a tenant. if pushStats.numLines != 0 && userID != "" { linesIngested.WithLabelValues(userID).Add(float64(pushStats.numLines)) From d37343b9dc6f2c54de31ad71068ea7274362cf13 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 1 Feb 2024 17:03:30 +0100 Subject: [PATCH 06/31] Start tracking discarded bytes --- pkg/distributor/distributor.go | 25 +++++++++++++++++-------- pkg/distributor/distributor_test.go | 2 +- pkg/distributor/validator.go | 24 +++++++++++++++++++++++- pkg/distributor/validator_test.go | 17 +++++++++-------- pkg/ingester/instance.go | 29 +++++++++++++++++------------ pkg/loghttp/push/push.go | 21 ++++++++++----------- pkg/loghttp/push/push_test.go | 4 ++-- pkg/validation/validate.go | 10 +++++++++- 8 files changed, 88 insertions(+), 44 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index a5229b0ca1498..85d02d9f1b536 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -337,7 +337,8 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log // Truncate first so subsequent steps have consistent line lengths d.truncateLines(validationContext, &stream) - stream.Labels, stream.Hash, err = d.parseStreamLabels(validationContext, stream.Labels, &stream) + var lbs labels.Labels + lbs, stream.Labels, stream.Hash, err = d.parseStreamLabels(validationContext, stream.Labels, &stream) if err != nil { d.writeFailuresManager.Log(tenantID, err) validationErrors.Add(err) @@ -347,6 +348,9 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log bytes += len(e.Line) } validation.DiscardedBytes.WithLabelValues(validation.InvalidLabels, tenantID).Add(float64(bytes)) + for _, tracker := range validationContext.customTrackerConfig.MatchTrackers(lbs) { + validation.DiscardedBytesCustom.WithLabelValues(validation.RateLimited, tenantID, tracker).Add(float64(validatedLineSize)) + } continue } @@ -354,7 +358,7 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log pushSize := 0 prevTs := stream.Entries[0].Timestamp for _, entry := range stream.Entries { - if err := d.validator.ValidateEntry(validationContext, stream.Labels, entry); err != nil { + if err := d.validator.ValidateEntry(validationContext, lbs, entry); err != nil { d.writeFailuresManager.Log(tenantID, err) validationErrors.Add(err) continue @@ -412,6 +416,10 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log validation.DiscardedSamples.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineCount)) validation.DiscardedBytes.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineSize)) + for _, tracker := range validationContext.customTrackerConfig.MatchTrackers(lbs) { + validation.DiscardedBytesCustom.WithLabelValues(validation.RateLimited, tenantID, tracker).Add(float64(validatedLineSize)) + } + err = fmt.Errorf(validation.RateLimitedErrorMsg, tenantID, int(d.ingestionRateLimiter.Limit(now, tenantID)), validatedLineCount, validatedLineSize) d.writeFailuresManager.Log(tenantID, err) return nil, httpgrpc.Errorf(http.StatusTooManyRequests, err.Error()) @@ -684,30 +692,31 @@ func (d *Distributor) sendStreamsErr(ctx context.Context, ingester ring.Instance } type labelData struct { + ls labels.Labels labels string hash uint64 } -func (d *Distributor) parseStreamLabels(vContext validationContext, key string, stream *logproto.Stream) (string, uint64, error) { +func (d *Distributor) parseStreamLabels(vContext validationContext, key string, stream *logproto.Stream) (labels.Labels, string, uint64, error) { if val, ok := d.labelCache.Get(key); ok { labelVal := val.(labelData) - return labelVal.labels, labelVal.hash, nil + return labelVal.ls, labelVal.labels, labelVal.hash, nil } ls, err := syntax.ParseLabels(key) if err != nil { - return "", 0, fmt.Errorf(validation.InvalidLabelsErrorMsg, key, err) + return nil, "", 0, fmt.Errorf(validation.InvalidLabelsErrorMsg, key, err) } if err := d.validator.ValidateLabels(vContext, ls, *stream); err != nil { - return "", 0, err + return nil, "", 0, err } lsVal := ls.String() lsHash := ls.Hash() - d.labelCache.Add(key, labelData{lsVal, lsHash}) - return lsVal, lsHash, nil + d.labelCache.Add(key, labelData{ls, lsVal, lsHash}) + return ls, lsVal, lsHash, nil } // shardCountFor returns the right number of shards to be used by the given stream. diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 71830b4be4d2e..1fdcaa0f1b346 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -782,7 +782,7 @@ func Benchmark_SortLabelsOnPush(b *testing.B) { for n := 0; n < b.N; n++ { stream := request.Streams[0] stream.Labels = `{buzz="f", a="b"}` - _, _, err := d.parseStreamLabels(vCtx, stream.Labels, &stream) + _, _, _, err := d.parseStreamLabels(vCtx, stream.Labels, &stream) if err != nil { panic("parseStreamLabels fail,err:" + err.Error()) } diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index 7fe76fae78231..1b068e4371c7a 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -8,6 +8,7 @@ import ( "github.com/prometheus/prometheus/model/labels" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/validation" ) @@ -46,6 +47,8 @@ type validationContext struct { maxStructuredMetadataCount int userID string + + customTrackerConfig push.CustomTrackersConfig } func (v Validator) getValidationContextForTime(now time.Time, userID string) validationContext { @@ -63,11 +66,12 @@ func (v Validator) getValidationContextForTime(now time.Time, userID string) val allowStructuredMetadata: v.AllowStructuredMetadata(userID), maxStructuredMetadataSize: v.MaxStructuredMetadataSize(userID), maxStructuredMetadataCount: v.MaxStructuredMetadataCount(userID), + customTrackerConfig: v.CustomTrackersConfig(userID), } } // ValidateEntry returns an error if the entry is invalid and report metrics for invalid entries accordingly. -func (v Validator) ValidateEntry(ctx validationContext, labels string, entry logproto.Entry) error { +func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, entry logproto.Entry) error { ts := entry.Timestamp.UnixNano() validation.LineLengthHist.Observe(float64(len(entry.Line))) @@ -77,6 +81,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log formatedRejectMaxAgeTime := time.Unix(0, ctx.rejectOldSampleMaxAge).Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Add(float64(len(entry.Line))) + for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { + validation.DiscardedBytesCustom.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID, tracker).Add(float64(len(entry.Line))) + } return fmt.Errorf(validation.GreaterThanMaxSampleAgeErrorMsg, labels, formatedEntryTime, formatedRejectMaxAgeTime) } @@ -84,6 +91,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log formatedEntryTime := entry.Timestamp.Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.TooFarInFuture, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.TooFarInFuture, ctx.userID).Add(float64(len(entry.Line))) + for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { + validation.DiscardedBytesCustom.WithLabelValues(validation.TooFarInFuture, ctx.userID, tracker).Add(float64(len(entry.Line))) + } return fmt.Errorf(validation.TooFarInFutureErrorMsg, labels, formatedEntryTime) } @@ -94,6 +104,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log // for parity. validation.DiscardedSamples.WithLabelValues(validation.LineTooLong, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.LineTooLong, ctx.userID).Add(float64(len(entry.Line))) + for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { + validation.DiscardedBytesCustom.WithLabelValues(validation.LineTooLong, ctx.userID, tracker).Add(float64(len(entry.Line))) + } return fmt.Errorf(validation.LineTooLongErrorMsg, maxSize, labels, len(entry.Line)) } @@ -101,6 +114,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log if !ctx.allowStructuredMetadata { validation.DiscardedSamples.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Add(float64(len(entry.Line))) + for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { + validation.DiscardedBytesCustom.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID, tracker).Add(float64(len(entry.Line))) + } return fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, labels) } @@ -113,12 +129,18 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log if maxSize := ctx.maxStructuredMetadataSize; maxSize != 0 && structuredMetadataSizeBytes > maxSize { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Add(float64(len(entry.Line))) + for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { + validation.DiscardedBytesCustom.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID, tracker).Add(float64(len(entry.Line))) + } return fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, labels, structuredMetadataSizeBytes, ctx.maxStructuredMetadataSize) } if maxCount := ctx.maxStructuredMetadataCount; maxCount != 0 && structuredMetadataCount > maxCount { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Add(float64(len(entry.Line))) + for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { + validation.DiscardedBytesCustom.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID, tracker).Add(float64(len(entry.Line))) + } return fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, labels, structuredMetadataCount, ctx.maxStructuredMetadataCount) } } diff --git a/pkg/distributor/validator_test.go b/pkg/distributor/validator_test.go index 038f1dc4c5b78..185e3a9f86b8d 100644 --- a/pkg/distributor/validator_test.go +++ b/pkg/distributor/validator_test.go @@ -18,8 +18,9 @@ import ( ) var ( - testStreamLabels = "FIXME" - testTime = time.Now() + testStreamLabels = labels.Labels{{Name:"my", Value:"label"}} + testStreamLabelsString = testStreamLabels.String() + testTime = time.Now() ) type fakeLimits struct { @@ -61,7 +62,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, logproto.Entry{Timestamp: testTime.Add(-time.Hour * 5), Line: "test"}, fmt.Errorf(validation.GreaterThanMaxSampleAgeErrorMsg, - testStreamLabels, + testStreamLabelsString, testTime.Add(-time.Hour*5).Format(timeFormat), testTime.Add(-1*time.Hour).Format(timeFormat), // same as RejectOldSamplesMaxAge ), @@ -71,7 +72,7 @@ func TestValidator_ValidateEntry(t *testing.T) { "test", nil, logproto.Entry{Timestamp: testTime.Add(time.Hour * 5), Line: "test"}, - fmt.Errorf(validation.TooFarInFutureErrorMsg, testStreamLabels, testTime.Add(time.Hour*5).Format(timeFormat)), + fmt.Errorf(validation.TooFarInFutureErrorMsg, testStreamLabelsString, testTime.Add(time.Hour*5).Format(timeFormat)), }, { "line too long", @@ -82,7 +83,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901"}, - fmt.Errorf(validation.LineTooLongErrorMsg, 10, testStreamLabels, 11), + fmt.Errorf(validation.LineTooLongErrorMsg, 10, testStreamLabelsString, 11), }, { "disallowed structured metadata", @@ -93,7 +94,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901", StructuredMetadata: push.LabelsAdapter{{Name: "foo", Value: "bar"}}}, - fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, testStreamLabels), + fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, testStreamLabelsString), }, { "structured metadata too big", @@ -105,7 +106,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901", StructuredMetadata: push.LabelsAdapter{{Name: "foo", Value: "bar"}}}, - fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, testStreamLabels, 6, 4), + fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, testStreamLabelsString, 6, 4), }, { "structured metadata too many", @@ -117,7 +118,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901", StructuredMetadata: push.LabelsAdapter{{Name: "foo", Value: "bar"}, {Name: "too", Value: "many"}}}, - fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, testStreamLabels, 2, 1), + fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, testStreamLabelsString, 2, 1), }, } for _, tt := range tests { diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 4521daaf20123..aad5707539140 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -262,6 +262,20 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *wal.Recor // record is only nil when replaying WAL. We don't want to drop data when replaying a WAL after // reducing the stream limits, for instance. var err error + + labels, err := syntax.ParseLabels(pushReqStream.Labels) + if err != nil { + if i.configs.LogStreamCreation(i.instanceID) { + level.Debug(util_log.Logger).Log( + "msg", "failed to create stream, failed to parse labels", + "org_id", i.instanceID, + "err", err, + "stream", pushReqStream.Labels, + ) + } + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + if record != nil { err = i.limiter.AssertMaxStreamsPerUser(i.instanceID, i.streams.Len()) } @@ -282,21 +296,12 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *wal.Recor bytes += len(e.Line) } validation.DiscardedBytes.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(bytes)) + for _, tracker := range []string{} { // TODO: call customTracker.MatchedTrackers(lbs) + validation.DiscardedBytesCustom.WithLabelValues(validation.StreamLimit, i.instanceID, tracker).Add(float64(bytes)) + } return nil, httpgrpc.Errorf(http.StatusTooManyRequests, validation.StreamLimitErrorMsg, i.instanceID) } - labels, err := syntax.ParseLabels(pushReqStream.Labels) - if err != nil { - if i.configs.LogStreamCreation(i.instanceID) { - level.Debug(util_log.Logger).Log( - "msg", "failed to create stream, failed to parse labels", - "org_id", i.instanceID, - "err", err, - "stream", pushReqStream.Labels, - ) - } - return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) - } fp := i.getHashForLabels(labels) sortedLabels := i.index.Add(logproto.FromLabelsToLabelAdapters(labels), fp) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 37918300c8045..cee1a3366480c 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -227,12 +227,18 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe for _, s := range req.Streams { pushStats.streamLabelsSize += int64(len(s.Labels)) - var retentionPeriod time.Duration - if tenantsRetention != nil { - lbs, err := syntax.ParseLabels(s.Labels) + + customTrackers := limits.CustomTrackersConfig(userID) + var lbs labels.Labels + if tenantsRetention != nil || len(customTrackers.config) > 0 { + lbs, err = syntax.ParseLabels(s.Labels) if err != nil { return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) } + } + + var retentionPeriod time.Duration + if tenantsRetention != nil { retentionPeriod = tenantsRetention.RetentionPeriodFor(userID, lbs) } for _, e := range s.Entries { @@ -247,14 +253,7 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe pushStats.mostRecentEntryTimestamp = e.Timestamp } - trackers := limits.CustomTrackersConfig(userID) - - // TODO: do not duplicate - lbs, err := syntax.ParseLabels(s.Labels) - if err != nil { - return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) - } - for _, t := range trackers.MatchTrackers(lbs) { + for _, t := range customTrackers.MatchTrackers(lbs) { logLinesBytes, ok := pushStats.structuredMetadataBytesCustomTrackers[t] if !ok { pushStats.structuredMetadataBytesCustomTrackers[t] = map[time.Duration]int64{} diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index e495622ec0715..d2807ee70df17 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -236,11 +236,11 @@ func TestParseRequest(t *testing.T) { type mockLimits struct{} // CustomTrackersConfig implements Limits. -func (*mockLimits) CustomTrackersConfig(userID string) CustomTrackersConfig { +func (*mockLimits) CustomTrackersConfig(string) CustomTrackersConfig { return CustomTrackersConfig{} } // OTLPConfig implements Limits. -func (*mockLimits) OTLPConfig(userID string) OTLPConfig { +func (*mockLimits) OTLPConfig(string) OTLPConfig { return DefaultOTLPConfig } diff --git a/pkg/validation/validate.go b/pkg/validation/validate.go index 8b93a2e906a2e..893a0ec3f9bbf 100644 --- a/pkg/validation/validate.go +++ b/pkg/validation/validate.go @@ -102,7 +102,6 @@ var MutatedBytes = promauto.NewCounterVec( []string{ReasonLabel, "truncated"}, ) -// TODO: add custom tracker // DiscardedBytes is a metric of the total discarded bytes, by reason. var DiscardedBytes = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -113,6 +112,15 @@ var DiscardedBytes = promauto.NewCounterVec( []string{ReasonLabel, "tenant"}, ) +var DiscardedBytesCustom = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: constants.Loki, + Name: "discarded_bytes_custom_total", + Help: "The total number of bytes that were discarded for tracked streams.", + }, + []string{ReasonLabel, "tenant", "tracker"}, +) + // DiscardedSamples is a metric of the number of discarded samples, by reason. var DiscardedSamples = promauto.NewCounterVec( prometheus.CounterOpts{ From d5be74f44a662796d02b0cd7320d194188f8e093 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 12:03:57 +0100 Subject: [PATCH 07/31] Calrify todo --- pkg/logql/syntax/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logql/syntax/parser.go b/pkg/logql/syntax/parser.go index bd5ffebc3cdb0..0ad6a304da284 100644 --- a/pkg/logql/syntax/parser.go +++ b/pkg/logql/syntax/parser.go @@ -252,7 +252,7 @@ func ParseLogSelector(input string, validate bool) (LogSelectorExpr, error) { // ParseLabels parses labels from a string using logql parser. func ParseLabels(lbs string) (labels.Labels, error) { - // TODO: I wonder if we should use memoization here. + // TODO: I wonder if we should use memoization here. There's a cache in ls, err := promql_parser.ParseMetric(lbs) if err != nil { return nil, err From 77efea5975e56392f7b8363e73e35965f3a15a71 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 13:01:16 +0100 Subject: [PATCH 08/31] Change return type --- pkg/distributor/limits.go | 2 +- pkg/distributor/validator.go | 2 +- pkg/ingester/instance.go | 2 +- pkg/ingester/limiter.go | 6 ++++++ pkg/loghttp/push/push.go | 2 +- pkg/loghttp/push/push_test.go | 4 ++-- pkg/validation/limits.go | 4 ++-- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/distributor/limits.go b/pkg/distributor/limits.go index 4c3416d1242b9..6d4c8080b49a5 100644 --- a/pkg/distributor/limits.go +++ b/pkg/distributor/limits.go @@ -31,5 +31,5 @@ type Limits interface { MaxStructuredMetadataSize(userID string) int MaxStructuredMetadataCount(userID string) int OTLPConfig(userID string) push.OTLPConfig - CustomTrackersConfig(userID string) push.CustomTrackersConfig + CustomTrackersConfig(userID string) *push.CustomTrackersConfig } diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index 1b068e4371c7a..f23856f0bbcc8 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -48,7 +48,7 @@ type validationContext struct { userID string - customTrackerConfig push.CustomTrackersConfig + customTrackerConfig *push.CustomTrackersConfig } func (v Validator) getValidationContextForTime(now time.Time, userID string) validationContext { diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index aad5707539140..3f92f40e0df6f 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -296,7 +296,7 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *wal.Recor bytes += len(e.Line) } validation.DiscardedBytes.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(bytes)) - for _, tracker := range []string{} { // TODO: call customTracker.MatchedTrackers(lbs) + for _, tracker := range i.limiter.CustumTrackersConfig(i.instanceID).MatchTrackers(labels) { validation.DiscardedBytesCustom.WithLabelValues(validation.StreamLimit, i.instanceID, tracker).Add(float64(bytes)) } return nil, httpgrpc.Errorf(http.StatusTooManyRequests, validation.StreamLimitErrorMsg, i.instanceID) diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index e48c2a018d277..599ae417b1474 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -9,6 +9,7 @@ import ( "golang.org/x/time/rate" "github.com/grafana/loki/pkg/distributor/shardstreams" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/validation" ) @@ -28,6 +29,7 @@ type Limits interface { MaxGlobalStreamsPerUser(userID string) int PerStreamRateLimit(userID string) validation.RateLimit ShardStreams(userID string) *shardstreams.Config + CustomTrackersConfig(userID string) *push.CustomTrackersConfig } // Limiter implements primitives to get the maximum number of streams @@ -139,6 +141,10 @@ func (l *Limiter) minNonZero(first, second int) int { return first } +func (l *Limiter) CustumTrackersConfig(userID string) *push.CustomTrackersConfig { + return l.limits.CustomTrackersConfig(userID) +} + type RateLimiterStrategy interface { RateLimit(tenant string) validation.RateLimit } diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index cee1a3366480c..0a82d47fc4117 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -67,7 +67,7 @@ type TenantsRetention interface { type Limits interface { OTLPConfig(userID string) OTLPConfig - CustomTrackersConfig(userID string) CustomTrackersConfig + CustomTrackersConfig(userID string) *CustomTrackersConfig } type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index d2807ee70df17..9df4d7408c9ea 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -236,8 +236,8 @@ func TestParseRequest(t *testing.T) { type mockLimits struct{} // CustomTrackersConfig implements Limits. -func (*mockLimits) CustomTrackersConfig(string) CustomTrackersConfig { - return CustomTrackersConfig{} +func (*mockLimits) CustomTrackersConfig(string) *CustomTrackersConfig { + return &CustomTrackersConfig{} } // OTLPConfig implements Limits. diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index e4b9f1607ca5c..0181fe8480bc4 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -202,7 +202,7 @@ type Limits struct { MaxStructuredMetadataEntriesCount int `yaml:"max_structured_metadata_entries_count" json:"max_structured_metadata_entries_count" doc:"description=Maximum number of structured metadata entries per log line."` OTLPConfig push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"` - CustomTrackersConfig push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes."` + CustomTrackersConfig *push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes."` } type StreamRetention struct { @@ -904,7 +904,7 @@ func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig { return o.getOverridesForUser(userID).OTLPConfig } -func (o *Overrides) CustomTrackersConfig(userID string) push.CustomTrackersConfig { +func (o *Overrides) CustomTrackersConfig(userID string) *push.CustomTrackersConfig { return o.getOverridesForUser(userID).CustomTrackersConfig } From 51cbaa4cb25e4fdef7b4a07ee97f2a8e872db263 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 13:06:06 +0100 Subject: [PATCH 09/31] Comment discarded bytes --- pkg/distributor/distributor.go | 2 ++ pkg/distributor/validator_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 85d02d9f1b536..0a267535ae803 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -416,9 +416,11 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log validation.DiscardedSamples.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineCount)) validation.DiscardedBytes.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineSize)) + /* We would have to filter all streams again here. for _, tracker := range validationContext.customTrackerConfig.MatchTrackers(lbs) { validation.DiscardedBytesCustom.WithLabelValues(validation.RateLimited, tenantID, tracker).Add(float64(validatedLineSize)) } + */ err = fmt.Errorf(validation.RateLimitedErrorMsg, tenantID, int(d.ingestionRateLimiter.Limit(now, tenantID)), validatedLineCount, validatedLineSize) d.writeFailuresManager.Log(tenantID, err) diff --git a/pkg/distributor/validator_test.go b/pkg/distributor/validator_test.go index 185e3a9f86b8d..5656f24de1855 100644 --- a/pkg/distributor/validator_test.go +++ b/pkg/distributor/validator_test.go @@ -18,7 +18,7 @@ import ( ) var ( - testStreamLabels = labels.Labels{{Name:"my", Value:"label"}} + testStreamLabels = labels.Labels{{Name: "my", Value: "label"}} testStreamLabelsString = testStreamLabels.String() testTime = time.Now() ) From b0b4c7cde11fb89b63904b8b472a75278f653e5c Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 13:06:40 +0100 Subject: [PATCH 10/31] generate docs --- docs/sources/configure/_index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 25e4f70f987c3..401c6a72972f2 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -3151,6 +3151,9 @@ otlp_config: [scope_attributes: ] [log_attributes: ] + +# Defines a set of custom trackers for ingested bytes. +custom_trackers: ``` ### frontend_worker From fe34b32c37ec87139b9835d13e124cb7b224e288 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 13:09:12 +0100 Subject: [PATCH 11/31] Update docs --- docs/sources/configure/_index.md | 1 + pkg/validation/limits.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 401c6a72972f2..596f8ddc884ea 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -3153,6 +3153,7 @@ otlp_config: [log_attributes: ] # Defines a set of custom trackers for ingested bytes. +# Takes a map of tracker names as keys and stream selectors as values. custom_trackers: ``` diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 0181fe8480bc4..c93828a7edf4b 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -202,7 +202,7 @@ type Limits struct { MaxStructuredMetadataEntriesCount int `yaml:"max_structured_metadata_entries_count" json:"max_structured_metadata_entries_count" doc:"description=Maximum number of structured metadata entries per log line."` OTLPConfig push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"` - CustomTrackersConfig *push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes."` + CustomTrackersConfig *push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes.\nTakes a map of tracker names as keys and stream selectors as values."` } type StreamRetention struct { From be4d2c83f5a507fcaccbefeeb8384757431465c0 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 13:12:29 +0100 Subject: [PATCH 12/31] Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37599ae8d347c..24bd2b84d8a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ##### Enhancements +* [11840](https://github.com/grafana/loki/pull/11840) **jeschkies**: Allow custom trackers for ingested and discarded bytes metric. * [11633](https://github.com/grafana/loki/pull/11633) **cyriltovena**: Add profiling integrations to tracing instrumentation. * [11571](https://github.com/grafana/loki/pull/11571) **MichelHollands**: Add a metrics.go log line for requests from querier to ingester * [11477](https://github.com/grafana/loki/pull/11477) **MichelHollands**: support GET for /ingester/shutdown From 659a9efa9c343279e92907964f2b3acf8186e02b Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 13:36:41 +0100 Subject: [PATCH 13/31] Test push metrics --- pkg/distributor/distributor.go | 2 +- pkg/loghttp/push/otlp.go | 3 +++ pkg/loghttp/push/push.go | 4 ++-- pkg/loghttp/push/push_test.go | 40 +++++++++++++++++++++++----------- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 0a267535ae803..c3ecb87241040 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -349,7 +349,7 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log } validation.DiscardedBytes.WithLabelValues(validation.InvalidLabels, tenantID).Add(float64(bytes)) for _, tracker := range validationContext.customTrackerConfig.MatchTrackers(lbs) { - validation.DiscardedBytesCustom.WithLabelValues(validation.RateLimited, tenantID, tracker).Add(float64(validatedLineSize)) + validation.DiscardedBytesCustom.WithLabelValues(validation.RateLimited, tenantID, tracker).Add(float64(bytes)) } continue } diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go index c25477a984e24..94845c35bf6b0 100644 --- a/pkg/loghttp/push/otlp.go +++ b/pkg/loghttp/push/otlp.go @@ -40,6 +40,9 @@ func newPushStats() *Stats { return &Stats{ logLinesBytes: map[time.Duration]int64{}, structuredMetadataBytes: map[time.Duration]int64{}, + + logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, + structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, } } diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 0a82d47fc4117..12369d482e1df 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -37,7 +37,6 @@ var ( Help: "The total number of uncompressed bytes received per tenant. Includes structured metadata bytes.", }, []string{"tenant", "retention_hours"}) - // TODO: track discarded bytes as well bytesIngestedCustom = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_bytes_received_custom_tracker_total", @@ -236,6 +235,7 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) } } + trackers := customTrackers.MatchTrackers(lbs) var retentionPeriod time.Duration if tenantsRetention != nil { @@ -253,7 +253,7 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe pushStats.mostRecentEntryTimestamp = e.Timestamp } - for _, t := range customTrackers.MatchTrackers(lbs) { + for _, t := range trackers { logLinesBytes, ok := pushStats.structuredMetadataBytesCustomTrackers[t] if !ok { pushStats.structuredMetadataBytesCustomTrackers[t] = map[time.Duration]int64{} diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index 9df4d7408c9ea..f04743084a241 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -54,6 +54,8 @@ func TestParseRequest(t *testing.T) { expectedStructuredMetadataBytes int expectedBytes int expectedLines int + customTrackers map[string]string + expectedBytesCustomTracker map[string]int }{ { path: `/loki/api/v1/push`, @@ -76,13 +78,15 @@ func TestParseRequest(t *testing.T) { expectedLines: 1, }, { - path: `/loki/api/v1/push`, - body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, - contentType: `application/json`, - contentEncoding: ``, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, + contentType: `application/json`, + contentEncoding: ``, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + customTrackers: map[string]string{"t": `{foo=~"b.*2"}`}, + expectedBytesCustomTracker: map[string]int{"t": len("fizzbuss")}, }, { path: `/loki/api/v1/push`, @@ -190,6 +194,7 @@ func TestParseRequest(t *testing.T) { t.Run(fmt.Sprintf("test %d", index), func(t *testing.T) { structuredMetadataBytesIngested.Reset() bytesIngested.Reset() + bytesIngestedCustom.Reset() linesIngested.Reset() request := httptest.NewRequest("POST", test.path, strings.NewReader(test.body)) @@ -200,7 +205,11 @@ func TestParseRequest(t *testing.T) { request.Header.Add("Content-Encoding", test.contentEncoding) } - limits := &mockLimits{} + CustomTrackersConfig, err := NewCustomTrackersConfig(test.customTrackers) + require.NoError(t, err) + limits := &mockLimits{ + customTrackers: CustomTrackersConfig, + } data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest) structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived @@ -211,7 +220,7 @@ func TestParseRequest(t *testing.T) { previousLinesReceived += linesReceived if test.valid { - assert.Nil(t, err, "Should not give error for %d", index) + assert.NoErrorf(t, err, "Should not give error for %d", index) assert.NotNil(t, data, "Should give data for %d", index) require.Equal(t, test.expectedStructuredMetadataBytes, structuredMetadataBytesReceived) require.Equal(t, test.expectedBytes, bytesReceived) @@ -219,8 +228,11 @@ func TestParseRequest(t *testing.T) { require.Equal(t, float64(test.expectedStructuredMetadataBytes), testutil.ToFloat64(structuredMetadataBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedBytes), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedLines), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) + for tracker, value := range test.expectedBytesCustomTracker { + require.Equal(t, float64(value), testutil.ToFloat64(bytesIngestedCustom.WithLabelValues("fake", "", tracker))) + } } else { - assert.NotNil(t, err, "Should give error for %d", index) + assert.Errorf(t, err, "Should give error for %d", index) assert.Nil(t, data, "Should not give data for %d", index) require.Equal(t, 0, structuredMetadataBytesReceived) require.Equal(t, 0, bytesReceived) @@ -233,11 +245,13 @@ func TestParseRequest(t *testing.T) { } } -type mockLimits struct{} +type mockLimits struct { + customTrackers CustomTrackersConfig +} // CustomTrackersConfig implements Limits. -func (*mockLimits) CustomTrackersConfig(string) *CustomTrackersConfig { - return &CustomTrackersConfig{} +func (m *mockLimits) CustomTrackersConfig(string) *CustomTrackersConfig { + return &m.customTrackers } // OTLPConfig implements Limits. From 90099efb7df3180af3aa18578593cacc25227f88 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 14:25:16 +0100 Subject: [PATCH 14/31] Return empty trackers instead of nil --- pkg/loghttp/push/custom_trackers.go | 5 +++++ pkg/validation/limits.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index a52756f061b7a..4e7111cdc5ab1 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -13,6 +13,11 @@ type CustomTrackersConfig struct { config map[string][]*labels.Matcher } +var EmptyCustomTrackersConfig = &CustomTrackersConfig{ + source: map[string]string{}, + config: map[string][]*labels.Matcher{}, +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. // CustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. func (c *CustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index c93828a7edf4b..e0f42bcfd8611 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -905,6 +905,10 @@ func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig { } func (o *Overrides) CustomTrackersConfig(userID string) *push.CustomTrackersConfig { + if o.getOverridesForUser(userID).CustomTrackersConfig == nil { + return push.EmptyCustomTrackersConfig + } + return o.getOverridesForUser(userID).CustomTrackersConfig } From 4f0697a79fe7ebcd1504fd1695921466c135c35c Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 2 Feb 2024 16:49:26 +0100 Subject: [PATCH 15/31] Test oltp push --- pkg/loghttp/push/custom_trackers.go | 20 +++++-- pkg/loghttp/push/otlp.go | 11 +++- pkg/loghttp/push/otlp_test.go | 89 ++++++++++++++++++----------- pkg/loghttp/push/push_test.go | 8 +-- 4 files changed, 86 insertions(+), 42 deletions(-) diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index 4e7111cdc5ab1..8843e4ce9d124 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -26,13 +26,16 @@ func (c *CustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) if err != nil { return err } - *c, err = NewCustomTrackersConfig(stringMap) + tmp, err := NewCustomTrackersConfig(stringMap) + *c = *tmp return err } -func NewCustomTrackersConfig(m map[string]string) (c CustomTrackersConfig, err error) { - c.source = m - c.config = map[string][]*labels.Matcher{} +func NewCustomTrackersConfig(m map[string]string) (*CustomTrackersConfig, error) { + c := &CustomTrackersConfig{ + source: m, + config: map[string][]*labels.Matcher{}, + } for name, selector := range m { matchers, err := syntax.ParseMatchers(selector, true) if err != nil { @@ -43,6 +46,15 @@ func NewCustomTrackersConfig(m map[string]string) (c CustomTrackersConfig, err e return c, nil } +func MustNewCustomTrackersConfig(m map[string]string) *CustomTrackersConfig { + c, err := NewCustomTrackersConfig(m) + if err != nil { + panic(err) + } + + return c +} + // MatchTrackers returns a list of names of all trackers that match the given labels. func (c *CustomTrackersConfig) MatchTrackers(lbs labels.Labels) []string { trackers := make([]string, 0) diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go index 94845c35bf6b0..2b4d8d5f48d06 100644 --- a/pkg/loghttp/push/otlp.go +++ b/pkg/loghttp/push/otlp.go @@ -53,7 +53,7 @@ func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRe return nil, nil, err } - req := otlpToLokiPushRequest(otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), stats) + req := otlpToLokiPushRequest(otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), limits.CustomTrackersConfig(userID), stats) return req, stats, nil } @@ -104,7 +104,7 @@ func extractLogs(r *http.Request, pushStats *Stats) (plog.Logs, error) { return req.Logs(), nil } -func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, stats *Stats) *logproto.PushRequest { +func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, customTrackersConfig *CustomTrackersConfig, stats *Stats) *logproto.PushRequest { if ld.LogRecordCount() == 0 { return &logproto.PushRequest{} } @@ -148,6 +148,7 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants labelsStr := streamLabels.String() lbs := modelLabelsSetToLabelsList(streamLabels) + trackers := customTrackersConfig.MatchTrackers(lbs) if _, ok := pushRequestsByStream[labelsStr]; !ok { pushRequestsByStream[labelsStr] = logproto.Stream{ Labels: labelsStr, @@ -228,6 +229,12 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants stats.structuredMetadataBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) stats.logLinesBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) + for _, tracker := range trackers { + if _, ok := stats.logLinesBytesCustomTrackers[tracker]; !ok { + stats.logLinesBytesCustomTrackers[tracker] = map[time.Duration]int64{} + } + stats.logLinesBytesCustomTrackers[tracker][tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) + } stats.numLines++ if entry.Timestamp.After(stats.mostRecentEntryTimestamp) { stats.mostRecentEntryTimestamp = entry.Timestamp diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go index badb6cd000e4b..27acc1e1a1e40 100644 --- a/pkg/loghttp/push/otlp_test.go +++ b/pkg/loghttp/push/otlp_test.go @@ -20,20 +20,22 @@ func TestOTLPToLokiPushRequest(t *testing.T) { now := time.Unix(0, time.Now().UnixNano()) for _, tc := range []struct { - name string - generateLogs func() plog.Logs - expectedPushRequest logproto.PushRequest - expectedStats Stats - otlpConfig OTLPConfig + name string + generateLogs func() plog.Logs + expectedPushRequest logproto.PushRequest + expectedStats Stats + otlpConfig OTLPConfig + customTrackersConfig *CustomTrackersConfig }{ { name: "no logs", generateLogs: func() plog.Logs { return plog.NewLogs() }, - expectedPushRequest: logproto.PushRequest{}, - expectedStats: *newPushStats(), - otlpConfig: DefaultOTLPConfig, + expectedPushRequest: logproto.PushRequest{}, + expectedStats: *newPushStats(), + otlpConfig: DefaultOTLPConfig, + customTrackersConfig: EmptyCustomTrackersConfig, }, { name: "resource with no logs", @@ -42,13 +44,15 @@ func TestOTLPToLokiPushRequest(t *testing.T) { ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.name", "service-1") return ld }, - expectedPushRequest: logproto.PushRequest{}, - expectedStats: *newPushStats(), - otlpConfig: DefaultOTLPConfig, + expectedPushRequest: logproto.PushRequest{}, + expectedStats: *newPushStats(), + otlpConfig: DefaultOTLPConfig, + customTrackersConfig: EmptyCustomTrackersConfig, }, { - name: "resource with a log entry", - otlpConfig: DefaultOTLPConfig, + name: "resource with a log entry", + otlpConfig: DefaultOTLPConfig, + customTrackersConfig: EmptyCustomTrackersConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.name", "service-1") @@ -78,13 +82,16 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 0, }, - streamLabelsSize: 21, - mostRecentEntryTimestamp: now, + streamLabelsSize: 21, + mostRecentEntryTimestamp: now, + logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, + structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, }, }, { - name: "no resource attributes defined", - otlpConfig: DefaultOTLPConfig, + name: "no resource attributes defined", + otlpConfig: DefaultOTLPConfig, + customTrackersConfig: EmptyCustomTrackersConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -114,13 +121,16 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 0, }, - streamLabelsSize: 27, - mostRecentEntryTimestamp: now, + streamLabelsSize: 27, + mostRecentEntryTimestamp: now, + logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, + structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, }, }, { - name: "service.name not defined in resource attributes", - otlpConfig: DefaultOTLPConfig, + name: "service.name not defined in resource attributes", + otlpConfig: DefaultOTLPConfig, + customTrackersConfig: MustNewCustomTrackersConfig(map[string]string{"foo": `{service_namespace="foo"}`}), generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.namespace", "foo") @@ -152,11 +162,18 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 47, mostRecentEntryTimestamp: now, + logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{ + "foo": { + time.Hour: 9, + }, + }, + structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, }, }, { - name: "resource attributes and scope attributes stored as structured metadata", - otlpConfig: DefaultOTLPConfig, + name: "resource attributes and scope attributes stored as structured metadata", + otlpConfig: DefaultOTLPConfig, + customTrackersConfig: EmptyCustomTrackersConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -225,13 +242,16 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 37, }, - streamLabelsSize: 21, - mostRecentEntryTimestamp: now, + streamLabelsSize: 21, + mostRecentEntryTimestamp: now, + logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, + structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, }, }, { - name: "attributes with nested data", - otlpConfig: DefaultOTLPConfig, + name: "attributes with nested data", + otlpConfig: DefaultOTLPConfig, + customTrackersConfig: EmptyCustomTrackersConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -309,8 +329,10 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 97, }, - streamLabelsSize: 21, - mostRecentEntryTimestamp: now, + streamLabelsSize: 21, + mostRecentEntryTimestamp: now, + logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, + structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, }, }, { @@ -352,6 +374,7 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, }, }, + customTrackersConfig: EmptyCustomTrackersConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -452,14 +475,16 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 113, }, - streamLabelsSize: 42, - mostRecentEntryTimestamp: now, + streamLabelsSize: 42, + mostRecentEntryTimestamp: now, + logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, + structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, }, }, } { t.Run(tc.name, func(t *testing.T) { stats := newPushStats() - pushReq := otlpToLokiPushRequest(tc.generateLogs(), "foo", fakeRetention{}, tc.otlpConfig, stats) + pushReq := otlpToLokiPushRequest(tc.generateLogs(), "foo", fakeRetention{}, tc.otlpConfig, tc.customTrackersConfig, stats) require.Equal(t, tc.expectedPushRequest, *pushReq) require.Equal(t, tc.expectedStats, *stats) }) diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index f04743084a241..b0e8fc6e66664 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -205,10 +205,10 @@ func TestParseRequest(t *testing.T) { request.Header.Add("Content-Encoding", test.contentEncoding) } - CustomTrackersConfig, err := NewCustomTrackersConfig(test.customTrackers) + customTrackersConfig, err := NewCustomTrackersConfig(test.customTrackers) require.NoError(t, err) limits := &mockLimits{ - customTrackers: CustomTrackersConfig, + customTrackers: customTrackersConfig, } data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest) @@ -246,12 +246,12 @@ func TestParseRequest(t *testing.T) { } type mockLimits struct { - customTrackers CustomTrackersConfig + customTrackers *CustomTrackersConfig } // CustomTrackersConfig implements Limits. func (m *mockLimits) CustomTrackersConfig(string) *CustomTrackersConfig { - return &m.customTrackers + return m.customTrackers } // OTLPConfig implements Limits. From 4c3292dba7fb5f3a06dc21efac6a7f9fa57876f9 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Tue, 20 Feb 2024 17:36:51 +0100 Subject: [PATCH 16/31] Use label names intead of regex matchers --- pkg/loghttp/push/custom_trackers.go | 46 +++++++---------------------- pkg/loghttp/push/otlp_test.go | 2 +- pkg/loghttp/push/push.go | 2 +- pkg/loghttp/push/push_test.go | 7 ++--- 4 files changed, 16 insertions(+), 41 deletions(-) diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index 8843e4ce9d124..8d85464437dbf 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -1,70 +1,46 @@ package push import ( - "fmt" - "github.com/prometheus/prometheus/model/labels" - - "github.com/grafana/loki/pkg/logql/syntax" ) type CustomTrackersConfig struct { - source map[string]string - config map[string][]*labels.Matcher + source map[string][]string } var EmptyCustomTrackersConfig = &CustomTrackersConfig{ - source: map[string]string{}, - config: map[string][]*labels.Matcher{}, + source: map[string][]string{}, } // UnmarshalYAML implements the yaml.Unmarshaler interface. // CustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. func (c *CustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - stringMap := map[string]string{} + stringMap := map[string][]string{} err := unmarshal(&stringMap) if err != nil { return err } - tmp, err := NewCustomTrackersConfig(stringMap) - *c = *tmp - return err + *c = *NewCustomTrackersConfig(stringMap) + return nil } -func NewCustomTrackersConfig(m map[string]string) (*CustomTrackersConfig, error) { - c := &CustomTrackersConfig{ +func NewCustomTrackersConfig(m map[string][]string) *CustomTrackersConfig { + return &CustomTrackersConfig{ source: m, - config: map[string][]*labels.Matcher{}, } - for name, selector := range m { - matchers, err := syntax.ParseMatchers(selector, true) - if err != nil { - return c, fmt.Errorf("invalid labels matchers: %w", err) - } - c.config[name] = matchers - } - return c, nil -} - -func MustNewCustomTrackersConfig(m map[string]string) *CustomTrackersConfig { - c, err := NewCustomTrackersConfig(m) - if err != nil { - panic(err) - } - - return c } // MatchTrackers returns a list of names of all trackers that match the given labels. func (c *CustomTrackersConfig) MatchTrackers(lbs labels.Labels) []string { trackers := make([]string, 0) Outer: - for name, matchers := range c.config { - for _, m := range matchers { - if !m.Matches(lbs.Get(m.Name)) { + for name, labels := range c.source { + for _, label := range labels { + if !lbs.Has(label) { continue Outer } } + // TODO: add label names trackers = append(trackers, name) } return trackers diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go index 27acc1e1a1e40..35f77f6d17fe8 100644 --- a/pkg/loghttp/push/otlp_test.go +++ b/pkg/loghttp/push/otlp_test.go @@ -130,7 +130,7 @@ func TestOTLPToLokiPushRequest(t *testing.T) { { name: "service.name not defined in resource attributes", otlpConfig: DefaultOTLPConfig, - customTrackersConfig: MustNewCustomTrackersConfig(map[string]string{"foo": `{service_namespace="foo"}`}), + customTrackersConfig: NewCustomTrackersConfig(map[string][]string{"foo": {"service_namespace"}}), generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.namespace", "foo") diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 12369d482e1df..9aee605516ce2 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -229,7 +229,7 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe customTrackers := limits.CustomTrackersConfig(userID) var lbs labels.Labels - if tenantsRetention != nil || len(customTrackers.config) > 0 { + if tenantsRetention != nil || len(customTrackers.source) > 0 { lbs, err = syntax.ParseLabels(s.Labels) if err != nil { return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index b0e8fc6e66664..c478d364626c1 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -54,7 +54,7 @@ func TestParseRequest(t *testing.T) { expectedStructuredMetadataBytes int expectedBytes int expectedLines int - customTrackers map[string]string + customTrackers map[string][]string expectedBytesCustomTracker map[string]int }{ { @@ -85,7 +85,7 @@ func TestParseRequest(t *testing.T) { valid: true, expectedBytes: len("fizzbuzz"), expectedLines: 1, - customTrackers: map[string]string{"t": `{foo=~"b.*2"}`}, + customTrackers: map[string][]string{"t": {"foo="}}, expectedBytesCustomTracker: map[string]int{"t": len("fizzbuss")}, }, { @@ -205,8 +205,7 @@ func TestParseRequest(t *testing.T) { request.Header.Add("Content-Encoding", test.contentEncoding) } - customTrackersConfig, err := NewCustomTrackersConfig(test.customTrackers) - require.NoError(t, err) + customTrackersConfig := NewCustomTrackersConfig(test.customTrackers) limits := &mockLimits{ customTrackers: customTrackersConfig, } From ca417c2636b0a65accc2f83578ebc73231f5cd1c Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Tue, 20 Feb 2024 17:43:29 +0100 Subject: [PATCH 17/31] correct doc --- docs/sources/configure/_index.md | 3 ++- pkg/validation/limits.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 81fcdda76e928..f0d61041b4ea1 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -3236,7 +3236,8 @@ otlp_config: [log_attributes: ] # Defines a set of custom trackers for ingested bytes. -# Takes a map of tracker names as keys and stream selectors as values. +# Takes a map of tracker names as keys and a list of label names as values. A +# tracker matches a stream if all labels are present. custom_trackers: ``` diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 03b91740ad19f..36e95f682dbf7 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -207,7 +207,7 @@ type Limits struct { MaxStructuredMetadataEntriesCount int `yaml:"max_structured_metadata_entries_count" json:"max_structured_metadata_entries_count" doc:"description=Maximum number of structured metadata entries per log line."` OTLPConfig push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"` - CustomTrackersConfig *push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes.\nTakes a map of tracker names as keys and stream selectors as values."` + CustomTrackersConfig *push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes.\nTakes a map of tracker names as keys and a list of label names as values. A tracker matches a stream if all labels are present."` } type StreamRetention struct { From ee947b39c006230e9900098fc8a95a78271b8d02 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Tue, 20 Feb 2024 17:52:41 +0100 Subject: [PATCH 18/31] Test unmarshalling --- pkg/loghttp/push/custom_trackers_test.go | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 pkg/loghttp/push/custom_trackers_test.go diff --git a/pkg/loghttp/push/custom_trackers_test.go b/pkg/loghttp/push/custom_trackers_test.go new file mode 100644 index 0000000000000..1141071c90cf8 --- /dev/null +++ b/pkg/loghttp/push/custom_trackers_test.go @@ -0,0 +1,25 @@ +package push + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +func Test_Unmarshal(t *testing.T) { + raw := ` +dev: + - region + - zone +integrations: + - job +` + var out CustomTrackersConfig + err := yaml.Unmarshal([]byte(raw), &out) + require.NoError(t, err) + require.Contains(t, out.source, "dev") + require.Contains(t, out.source, "integrations") + require.ElementsMatch(t, out.source["dev"], []string{"region", "zone"}) + require.ElementsMatch(t, out.source["integrations"], []string{"job"}) +} From e650cf0e4c0e4c611f61681697959849e5732451 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 21 Feb 2024 11:23:36 +0100 Subject: [PATCH 19/31] Introduce custom tracker interface --- pkg/distributor/distributor.go | 7 ++++-- pkg/distributor/validator.go | 39 +++++++++++++++++++---------- pkg/ingester/instance.go | 9 +++++-- pkg/loghttp/push/custom_trackers.go | 22 ++++++++++------ pkg/validation/validate.go | 9 ------- 5 files changed, 53 insertions(+), 33 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index c8d7a1c0f4b62..e5dae9ba742d4 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -39,6 +39,7 @@ import ( "github.com/grafana/loki/pkg/distributor/writefailures" "github.com/grafana/loki/pkg/ingester" "github.com/grafana/loki/pkg/ingester/client" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/runtime" @@ -126,6 +127,8 @@ type Distributor struct { ingesterAppendTimeouts *prometheus.CounterVec replicationFactor prometheus.Gauge streamShardCount prometheus.Counter + + customStreamsTracker push.CustomTracker } // New a distributor creates. @@ -348,8 +351,8 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log bytes += len(e.Line) } validation.DiscardedBytes.WithLabelValues(validation.InvalidLabels, tenantID).Add(float64(bytes)) - for _, tracker := range validationContext.customTrackerConfig.MatchTrackers(lbs) { - validation.DiscardedBytesCustom.WithLabelValues(validation.RateLimited, tenantID, tracker).Add(float64(bytes)) + for _, matchedLbs := range validationContext.customTrackerConfig.MatchTrackers(lbs) { + d.customStreamsTracker.DiscardedBytesAdd(tenantID, matchedLbs, float64(bytes)) } continue } diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index f23856f0bbcc8..caeb91ee3acaf 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -19,13 +19,14 @@ const ( type Validator struct { Limits + customStreamsTracker push.CustomTracker } func NewValidator(l Limits) (*Validator, error) { if l == nil { return nil, errors.New("nil Limits") } - return &Validator{l}, nil + return &Validator{l, nil}, nil } type validationContext struct { @@ -81,8 +82,10 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en formatedRejectMaxAgeTime := time.Unix(0, ctx.rejectOldSampleMaxAge).Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Add(float64(len(entry.Line))) - for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { - validation.DiscardedBytesCustom.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID, tracker).Add(float64(len(entry.Line))) + if v.customStreamsTracker != nil { + for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) + } } return fmt.Errorf(validation.GreaterThanMaxSampleAgeErrorMsg, labels, formatedEntryTime, formatedRejectMaxAgeTime) } @@ -91,8 +94,10 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en formatedEntryTime := entry.Timestamp.Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.TooFarInFuture, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.TooFarInFuture, ctx.userID).Add(float64(len(entry.Line))) - for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { - validation.DiscardedBytesCustom.WithLabelValues(validation.TooFarInFuture, ctx.userID, tracker).Add(float64(len(entry.Line))) + if v.customStreamsTracker != nil { + for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) + } } return fmt.Errorf(validation.TooFarInFutureErrorMsg, labels, formatedEntryTime) } @@ -104,8 +109,10 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en // for parity. validation.DiscardedSamples.WithLabelValues(validation.LineTooLong, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.LineTooLong, ctx.userID).Add(float64(len(entry.Line))) - for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { - validation.DiscardedBytesCustom.WithLabelValues(validation.LineTooLong, ctx.userID, tracker).Add(float64(len(entry.Line))) + if v.customStreamsTracker != nil { + for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) + } } return fmt.Errorf(validation.LineTooLongErrorMsg, maxSize, labels, len(entry.Line)) } @@ -114,8 +121,10 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en if !ctx.allowStructuredMetadata { validation.DiscardedSamples.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Add(float64(len(entry.Line))) - for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { - validation.DiscardedBytesCustom.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID, tracker).Add(float64(len(entry.Line))) + if v.customStreamsTracker != nil { + for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) + } } return fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, labels) } @@ -129,8 +138,10 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en if maxSize := ctx.maxStructuredMetadataSize; maxSize != 0 && structuredMetadataSizeBytes > maxSize { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Add(float64(len(entry.Line))) - for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { - validation.DiscardedBytesCustom.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID, tracker).Add(float64(len(entry.Line))) + if v.customStreamsTracker != nil { + for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) + } } return fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, labels, structuredMetadataSizeBytes, ctx.maxStructuredMetadataSize) } @@ -138,8 +149,10 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en if maxCount := ctx.maxStructuredMetadataCount; maxCount != 0 && structuredMetadataCount > maxCount { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Add(float64(len(entry.Line))) - for _, tracker := range ctx.customTrackerConfig.MatchTrackers(labels) { - validation.DiscardedBytesCustom.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID, tracker).Add(float64(len(entry.Line))) + if v.customStreamsTracker != nil { + for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) + } } return fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, labels, structuredMetadataCount, ctx.maxStructuredMetadataCount) } diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 10f2244e03b10..e8f70ed018436 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -30,6 +30,7 @@ import ( "github.com/grafana/loki/pkg/ingester/index" "github.com/grafana/loki/pkg/ingester/wal" "github.com/grafana/loki/pkg/iter" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" "github.com/grafana/loki/pkg/logql/log" @@ -119,6 +120,8 @@ type instance struct { writeFailures *writefailures.Manager schemaconfig *config.SchemaConfig + + customStreamsTracker push.CustomTracker } func newInstance( @@ -296,8 +299,10 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *wal.Recor bytes += len(e.Line) } validation.DiscardedBytes.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(bytes)) - for _, tracker := range i.limiter.CustumTrackersConfig(i.instanceID).MatchTrackers(labels) { - validation.DiscardedBytesCustom.WithLabelValues(validation.StreamLimit, i.instanceID, tracker).Add(float64(bytes)) + if i.customStreamsTracker != nil { + for _, matchedLbs := range i.limiter.CustumTrackersConfig(i.instanceID).MatchTrackers(labels) { + i.customStreamsTracker.DiscardedBytesAdd(i.instanceID, matchedLbs, float64(bytes)) + } } return nil, httpgrpc.Errorf(http.StatusTooManyRequests, validation.StreamLimitErrorMsg, i.instanceID) } diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index 8d85464437dbf..03fe8b73c5df9 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -4,6 +4,11 @@ import ( "github.com/prometheus/prometheus/model/labels" ) +type CustomTracker interface { + IngestedBytesAdd(tenant string, labels labels.Labels, value float64) + DiscardedBytesAdd(tenant string, labels labels.Labels, value float64) +} + type CustomTrackersConfig struct { source map[string][]string } @@ -31,17 +36,20 @@ func NewCustomTrackersConfig(m map[string][]string) *CustomTrackersConfig { } // MatchTrackers returns a list of names of all trackers that match the given labels. -func (c *CustomTrackersConfig) MatchTrackers(lbs labels.Labels) []string { - trackers := make([]string, 0) +func (c *CustomTrackersConfig) MatchTrackers(lbs labels.Labels) []labels.Labels { + trackers := make([]labels.Labels, 0) Outer: - for name, labels := range c.source { - for _, label := range labels { - if !lbs.Has(label) { + for tracker, requiredLbs := range c.source { + matchedLabels := make(labels.Labels, 0, len(requiredLbs)) + for _, name := range requiredLbs { + if value := lbs.Get(name); value != "" { + matchedLabels = append(matchedLabels, labels.Label{Name: name, Value: value}) + } else { continue Outer } } - // TODO: add label names - trackers = append(trackers, name) + matchedLabels = append(matchedLabels, labels.Label{Name: "tracker", Value: tracker}) + trackers = append(trackers, matchedLabels) } return trackers } diff --git a/pkg/validation/validate.go b/pkg/validation/validate.go index 893a0ec3f9bbf..09c444aa64987 100644 --- a/pkg/validation/validate.go +++ b/pkg/validation/validate.go @@ -112,15 +112,6 @@ var DiscardedBytes = promauto.NewCounterVec( []string{ReasonLabel, "tenant"}, ) -var DiscardedBytesCustom = promauto.NewCounterVec( - prometheus.CounterOpts{ - Namespace: constants.Loki, - Name: "discarded_bytes_custom_total", - Help: "The total number of bytes that were discarded for tracked streams.", - }, - []string{ReasonLabel, "tenant", "tracker"}, -) - // DiscardedSamples is a metric of the number of discarded samples, by reason. var DiscardedSamples = promauto.NewCounterVec( prometheus.CounterOpts{ From 4ce9aaf1cec801b0efe1b35a8f19767e2c41e1f7 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 21 Feb 2024 13:51:02 +0100 Subject: [PATCH 20/31] fix tests --- .../promtail/targets/lokipush/pushtarget.go | 2 +- pkg/distributor/http.go | 2 +- pkg/loghttp/push/custom_trackers.go | 4 +- pkg/loghttp/push/otlp.go | 26 +++++--- pkg/loghttp/push/otlp_test.go | 44 +++++++++----- pkg/loghttp/push/push.go | 60 ++++++++++--------- pkg/loghttp/push/push_test.go | 5 +- 7 files changed, 86 insertions(+), 57 deletions(-) diff --git a/clients/pkg/promtail/targets/lokipush/pushtarget.go b/clients/pkg/promtail/targets/lokipush/pushtarget.go index c981de0de3dda..14500e7be8526 100644 --- a/clients/pkg/promtail/targets/lokipush/pushtarget.go +++ b/clients/pkg/promtail/targets/lokipush/pushtarget.go @@ -111,7 +111,7 @@ func (t *PushTarget) run() error { func (t *PushTarget) handleLoki(w http.ResponseWriter, r *http.Request) { logger := util_log.WithContext(r.Context(), util_log.Logger) userID, _ := tenant.TenantID(r.Context()) - req, err := push.ParseRequest(logger, userID, r, nil, nil, push.ParseLokiRequest) + req, err := push.ParseRequest(logger, userID, r, nil, nil, push.ParseLokiRequest, nil) if err != nil { level.Warn(t.logger).Log("msg", "failed to parse incoming push request", "err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/pkg/distributor/http.go b/pkg/distributor/http.go index ce242355e077b..6b447b07ecf59 100644 --- a/pkg/distributor/http.go +++ b/pkg/distributor/http.go @@ -34,7 +34,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe http.Error(w, err.Error(), http.StatusBadRequest) return } - req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser) + req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser, d.customStreamsTracker) if err != nil { if d.tenantConfigs.LogPushRequest(tenantID) { level.Debug(logger).Log( diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index 03fe8b73c5df9..655fb854a11bc 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -1,11 +1,13 @@ package push import ( + "time" + "github.com/prometheus/prometheus/model/labels" ) type CustomTracker interface { - IngestedBytesAdd(tenant string, labels labels.Labels, value float64) + IngestedBytesAdd(tenant string, retentionPeriod time.Duration, labels labels.Labels, value float64) DiscardedBytesAdd(tenant string, labels labels.Labels, value float64) } diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go index 2b4d8d5f48d06..5d6c259987362 100644 --- a/pkg/loghttp/push/otlp.go +++ b/pkg/loghttp/push/otlp.go @@ -40,9 +40,6 @@ func newPushStats() *Stats { return &Stats{ logLinesBytes: map[time.Duration]int64{}, structuredMetadataBytes: map[time.Duration]int64{}, - - logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, - structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, } } @@ -148,7 +145,18 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants labelsStr := streamLabels.String() lbs := modelLabelsSetToLabelsList(streamLabels) - trackers := customTrackersConfig.MatchTrackers(lbs) + + // Init custom streams tracking + trackedLabels := customTrackersConfig.MatchTrackers(lbs) + stats.structuredMetadataBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) + stats.logLinesBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) + for i, labels := range trackedLabels { + stats.structuredMetadataBytesCustomTrackers[i].Labels = labels + stats.structuredMetadataBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} + stats.logLinesBytesCustomTrackers[i].Labels = labels + stats.logLinesBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} + } + if _, ok := pushRequestsByStream[labelsStr]; !ok { pushRequestsByStream[labelsStr] = logproto.Stream{ Labels: labelsStr, @@ -229,12 +237,12 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants stats.structuredMetadataBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) stats.logLinesBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) - for _, tracker := range trackers { - if _, ok := stats.logLinesBytesCustomTrackers[tracker]; !ok { - stats.logLinesBytesCustomTrackers[tracker] = map[time.Duration]int64{} - } - stats.logLinesBytesCustomTrackers[tracker][tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) + + for i := range trackedLabels { + stats.logLinesBytesCustomTrackers[i].Bytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) + stats.structuredMetadataBytesCustomTrackers[i].Bytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) } + stats.numLines++ if entry.Timestamp.After(stats.mostRecentEntryTimestamp) { stats.mostRecentEntryTimestamp = entry.Timestamp diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go index 35f77f6d17fe8..0bdc3f72f5c58 100644 --- a/pkg/loghttp/push/otlp_test.go +++ b/pkg/loghttp/push/otlp_test.go @@ -84,8 +84,8 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 21, mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, - structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, + logLinesBytesCustomTrackers: []customTrackerPair{}, + structuredMetadataBytesCustomTrackers: []customTrackerPair{}, }, }, { @@ -123,8 +123,8 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 27, mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, - structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, + logLinesBytesCustomTrackers: []customTrackerPair{}, + structuredMetadataBytesCustomTrackers: []customTrackerPair{}, }, }, { @@ -162,12 +162,28 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 47, mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{ - "foo": { - time.Hour: 9, + logLinesBytesCustomTrackers: []customTrackerPair{ + { + Labels: []labels.Label{ + {Name: "service_namespace", Value: "foo"}, + {Name: "tracker", Value: "foo"}, + }, + Bytes: map[time.Duration]int64{ + time.Hour: 9, + }, + }, + }, + structuredMetadataBytesCustomTrackers: []customTrackerPair{ + { + Labels: []labels.Label{ + {Name: "service_namespace", Value: "foo"}, + {Name: "tracker", Value: "foo"}, + }, + Bytes: map[time.Duration]int64{ + time.Hour: 0, + }, }, }, - structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, }, }, { @@ -244,8 +260,8 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 21, mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, - structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, + logLinesBytesCustomTrackers: []customTrackerPair{}, + structuredMetadataBytesCustomTrackers: []customTrackerPair{}, }, }, { @@ -331,8 +347,8 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 21, mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, - structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, + logLinesBytesCustomTrackers: []customTrackerPair{}, + structuredMetadataBytesCustomTrackers: []customTrackerPair{}, }, }, { @@ -477,8 +493,8 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 42, mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: map[string]map[time.Duration]int64{}, - structuredMetadataBytesCustomTrackers: map[string]map[time.Duration]int64{}, + logLinesBytesCustomTrackers: []customTrackerPair{}, + structuredMetadataBytesCustomTrackers: []customTrackerPair{}, }, }, } { diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 9aee605516ce2..697fd469d76cf 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -37,11 +37,6 @@ var ( Help: "The total number of uncompressed bytes received per tenant. Includes structured metadata bytes.", }, []string{"tenant", "retention_hours"}) - bytesIngestedCustom = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: constants.Loki, - Name: "distributor_bytes_received_custom_tracker_total", - Help: "The total number of uncompressed bytes received per tenant. Includes structured metadata bytes.", - }, []string{"tenant", "retention_hours", "tacker"}) structuredMetadataBytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_structured_metadata_bytes_received_total", @@ -71,6 +66,11 @@ type Limits interface { type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) +type customTrackerPair struct { + Labels labels.Labels + Bytes map[time.Duration]int64 +} + type Stats struct { errs []error numLines int64 @@ -82,11 +82,11 @@ type Stats struct { contentEncoding string bodySize int64 - logLinesBytesCustomTrackers map[string]map[time.Duration]int64 - structuredMetadataBytesCustomTrackers map[string]map[time.Duration]int64 + logLinesBytesCustomTrackers []customTrackerPair + structuredMetadataBytesCustomTrackers []customTrackerPair } -func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser) (*logproto.PushRequest, error) { +func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker CustomTracker) (*logproto.PushRequest, error) { req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits) if err != nil { return nil, err @@ -117,18 +117,16 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete } // Process custom trackers - for name, logLinesBytes := range pushStats.logLinesBytesCustomTrackers { - for retentionPeriod, size := range logLinesBytes { - retentionHours := retentionPeriodToString(retentionPeriod) - - bytesIngestedCustom.WithLabelValues(userID, retentionHours, name).Add(float64(size)) + if tracker != nil { + for _, pair := range pushStats.logLinesBytesCustomTrackers { + for retentionPeriod, size := range pair.Bytes { + tracker.IngestedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) + } } - } - for name, structuredMetadataBytes := range pushStats.structuredMetadataBytesCustomTrackers { - for retentionPeriod, size := range structuredMetadataBytes { - retentionHours := retentionPeriodToString(retentionPeriod) - - bytesIngestedCustom.WithLabelValues(userID, retentionHours, name).Add(float64(size)) + for _, pair := range pushStats.structuredMetadataBytesCustomTrackers { + for retentionPeriod, size := range pair.Bytes { + tracker.IngestedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) + } } } @@ -227,6 +225,7 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe for _, s := range req.Streams { pushStats.streamLabelsSize += int64(len(s.Labels)) + // Init custom trackers customTrackers := limits.CustomTrackersConfig(userID) var lbs labels.Labels if tenantsRetention != nil || len(customTrackers.source) > 0 { @@ -235,7 +234,13 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) } } - trackers := customTrackers.MatchTrackers(lbs) + trackedLabels := customTrackers.MatchTrackers(lbs) + // TODO: avoid allocation if if cap() <= len(trackerLabels) + pushStats.structuredMetadataBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) + for i, labels := range trackedLabels { + pushStats.structuredMetadataBytesCustomTrackers[i].Labels = labels + pushStats.structuredMetadataBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} + } var retentionPeriod time.Duration if tenantsRetention != nil { @@ -249,17 +254,14 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe } pushStats.logLinesBytes[retentionPeriod] += int64(len(e.Line)) pushStats.structuredMetadataBytes[retentionPeriod] += entryLabelsSize - if e.Timestamp.After(pushStats.mostRecentEntryTimestamp) { - pushStats.mostRecentEntryTimestamp = e.Timestamp + + for i := range trackedLabels { + pushStats.logLinesBytesCustomTrackers[i].Bytes[retentionPeriod] += int64(len(e.Line)) + pushStats.structuredMetadataBytesCustomTrackers[i].Bytes[retentionPeriod] += entryLabelsSize } - for _, t := range trackers { - logLinesBytes, ok := pushStats.structuredMetadataBytesCustomTrackers[t] - if !ok { - pushStats.structuredMetadataBytesCustomTrackers[t] = map[time.Duration]int64{} - logLinesBytes = pushStats.structuredMetadataBytesCustomTrackers[t] - } - logLinesBytes[retentionPeriod] += int64(len(e.Line)) + if e.Timestamp.After(pushStats.mostRecentEntryTimestamp) { + pushStats.mostRecentEntryTimestamp = e.Timestamp } } } diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index c478d364626c1..42f6468c67beb 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -194,7 +194,6 @@ func TestParseRequest(t *testing.T) { t.Run(fmt.Sprintf("test %d", index), func(t *testing.T) { structuredMetadataBytesIngested.Reset() bytesIngested.Reset() - bytesIngestedCustom.Reset() linesIngested.Reset() request := httptest.NewRequest("POST", test.path, strings.NewReader(test.body)) @@ -209,7 +208,7 @@ func TestParseRequest(t *testing.T) { limits := &mockLimits{ customTrackers: customTrackersConfig, } - data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest) + data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest, nil) // TODO: inject mocked custom tracker structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived previousStructuredMetadataBytesReceived += structuredMetadataBytesReceived @@ -227,9 +226,11 @@ func TestParseRequest(t *testing.T) { require.Equal(t, float64(test.expectedStructuredMetadataBytes), testutil.ToFloat64(structuredMetadataBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedBytes), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedLines), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) + /* TODO for tracker, value := range test.expectedBytesCustomTracker { require.Equal(t, float64(value), testutil.ToFloat64(bytesIngestedCustom.WithLabelValues("fake", "", tracker))) } + */ } else { assert.Errorf(t, err, "Should give error for %d", index) assert.Nil(t, data, "Should not give data for %d", index) From c57c493adca262ad1650564324c1db3168b4f48e Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 21 Feb 2024 15:43:30 +0100 Subject: [PATCH 21/31] Pass empty limits --- clients/pkg/promtail/targets/lokipush/pushtarget.go | 2 +- pkg/loghttp/push/push.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/clients/pkg/promtail/targets/lokipush/pushtarget.go b/clients/pkg/promtail/targets/lokipush/pushtarget.go index 14500e7be8526..88c7859bd36e5 100644 --- a/clients/pkg/promtail/targets/lokipush/pushtarget.go +++ b/clients/pkg/promtail/targets/lokipush/pushtarget.go @@ -111,7 +111,7 @@ func (t *PushTarget) run() error { func (t *PushTarget) handleLoki(w http.ResponseWriter, r *http.Request) { logger := util_log.WithContext(r.Context(), util_log.Logger) userID, _ := tenant.TenantID(r.Context()) - req, err := push.ParseRequest(logger, userID, r, nil, nil, push.ParseLokiRequest, nil) + req, err := push.ParseRequest(logger, userID, r, nil, push.EmptyLimits{}, push.ParseLokiRequest, nil) if err != nil { level.Warn(t.logger).Log("msg", "failed to parse incoming push request", "err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 697fd469d76cf..3cf4050c56f74 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -64,6 +64,16 @@ type Limits interface { CustomTrackersConfig(userID string) *CustomTrackersConfig } +type EmptyLimits struct{} + +func (EmptyLimits) OTLPConfig(string) OTLPConfig { + return DefaultOTLPConfig +} + +func (EmptyLimits) CustomTrackersConfig(string) *CustomTrackersConfig { + return EmptyCustomTrackersConfig +} + type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) type customTrackerPair struct { From 97a5af953c121309bab2705aed5d9b988730c370 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 21 Feb 2024 16:06:33 +0100 Subject: [PATCH 22/31] test tracker --- pkg/loghttp/push/push.go | 3 +++ pkg/loghttp/push/push_test.go | 35 ++++++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 3cf4050c56f74..8421a574a10b8 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -246,8 +246,11 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe } trackedLabels := customTrackers.MatchTrackers(lbs) // TODO: avoid allocation if if cap() <= len(trackerLabels) + pushStats.logLinesBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) pushStats.structuredMetadataBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) for i, labels := range trackedLabels { + pushStats.logLinesBytesCustomTrackers[i].Labels = labels + pushStats.logLinesBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} pushStats.structuredMetadataBytesCustomTrackers[i].Labels = labels pushStats.structuredMetadataBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} } diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index 42f6468c67beb..16dbe24763b90 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -9,8 +9,10 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -55,7 +57,7 @@ func TestParseRequest(t *testing.T) { expectedBytes int expectedLines int customTrackers map[string][]string - expectedBytesCustomTracker map[string]int + expectedBytesCustomTracker map[string]float64 }{ { path: `/loki/api/v1/push`, @@ -85,8 +87,8 @@ func TestParseRequest(t *testing.T) { valid: true, expectedBytes: len("fizzbuzz"), expectedLines: 1, - customTrackers: map[string][]string{"t": {"foo="}}, - expectedBytesCustomTracker: map[string]int{"t": len("fizzbuss")}, + customTrackers: map[string][]string{"t": {"foo"}}, + expectedBytesCustomTracker: map[string]float64{`{foo="bar2", tracker="t"}`: float64(len("fizzbuss"))}, }, { path: `/loki/api/v1/push`, @@ -208,7 +210,11 @@ func TestParseRequest(t *testing.T) { limits := &mockLimits{ customTrackers: customTrackersConfig, } - data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest, nil) // TODO: inject mocked custom tracker + tracker := &mockCustomTracker{ + receivedBytes: map[string]float64{}, + discardedBytes: map[string]float64{}, + } + data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest, tracker) structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived previousStructuredMetadataBytesReceived += structuredMetadataBytesReceived @@ -226,11 +232,7 @@ func TestParseRequest(t *testing.T) { require.Equal(t, float64(test.expectedStructuredMetadataBytes), testutil.ToFloat64(structuredMetadataBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedBytes), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedLines), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) - /* TODO - for tracker, value := range test.expectedBytesCustomTracker { - require.Equal(t, float64(value), testutil.ToFloat64(bytesIngestedCustom.WithLabelValues("fake", "", tracker))) - } - */ + require.InDeltaMapValues(t, test.expectedBytesCustomTracker, tracker.receivedBytes, 0.0) } else { assert.Errorf(t, err, "Should give error for %d", index) assert.Nil(t, data, "Should not give data for %d", index) @@ -258,3 +260,18 @@ func (m *mockLimits) CustomTrackersConfig(string) *CustomTrackersConfig { func (*mockLimits) OTLPConfig(string) OTLPConfig { return DefaultOTLPConfig } + +type mockCustomTracker struct { + receivedBytes map[string]float64 + discardedBytes map[string]float64 +} + +// DiscardedBytesAdd implements CustomTracker. +func (t *mockCustomTracker) DiscardedBytesAdd(_ string, labels labels.Labels, value float64) { + t.discardedBytes[labels.String()] += value +} + +// IngestedBytesAdd implements CustomTracker. +func (t *mockCustomTracker) IngestedBytesAdd(_ string, _ time.Duration, labels labels.Labels, value float64) { + t.receivedBytes[labels.String()] += value +} From 22e1d8ef42e0c1a2da5927ad4ac7236ce45cd5f9 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 22 Feb 2024 16:16:29 +0100 Subject: [PATCH 23/31] Report on discarded stream bytes --- pkg/distributor/distributor.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index e5dae9ba742d4..2b668585b0cea 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -351,9 +351,6 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log bytes += len(e.Line) } validation.DiscardedBytes.WithLabelValues(validation.InvalidLabels, tenantID).Add(float64(bytes)) - for _, matchedLbs := range validationContext.customTrackerConfig.MatchTrackers(lbs) { - d.customStreamsTracker.DiscardedBytesAdd(tenantID, matchedLbs, float64(bytes)) - } continue } @@ -419,11 +416,23 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log validation.DiscardedSamples.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineCount)) validation.DiscardedBytes.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineSize)) - /* We would have to filter all streams again here. - for _, tracker := range validationContext.customTrackerConfig.MatchTrackers(lbs) { - validation.DiscardedBytesCustom.WithLabelValues(validation.RateLimited, tenantID, tracker).Add(float64(validatedLineSize)) + if d.customStreamsTracker != nil { + for _, stream := range req.Streams { + lbs, _, _, err := d.parseStreamLabels(validationContext, stream.Labels, &stream) + if err != nil { + continue + } + + discardedStreamBytes := 0 + for _, e := range stream.Entries { + discardedStreamBytes += len(e.Line) + } + + for _, matchedLbs := range validationContext.customTrackerConfig.MatchTrackers(lbs) { + d.customStreamsTracker.DiscardedBytesAdd(tenantID, matchedLbs, float64(discardedStreamBytes)) + } + } } - */ err = fmt.Errorf(validation.RateLimitedErrorMsg, tenantID, int(d.ingestionRateLimiter.Limit(now, tenantID)), validatedLineCount, validatedLineSize) d.writeFailuresManager.Log(tenantID, err) From c564f409d5b34c6f97df72c2439bc8b3c0d37493 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 22 Feb 2024 16:25:58 +0100 Subject: [PATCH 24/31] Document custom tracker structs and methods --- pkg/loghttp/push/custom_trackers.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index 655fb854a11bc..9a542c6762be0 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -7,10 +7,15 @@ import ( ) type CustomTracker interface { + + // IngestedBytesAdd records ingested bytes by tenant, retention period and labels. IngestedBytesAdd(tenant string, retentionPeriod time.Duration, labels labels.Labels, value float64) + + // DiscardedBytesAdd records discarded bytes by tenant and labels. DiscardedBytesAdd(tenant string, labels labels.Labels, value float64) } +// CustomTrackersConfig defines a map of tracker name to tracked label set. type CustomTrackersConfig struct { source map[string][]string } @@ -37,7 +42,8 @@ func NewCustomTrackersConfig(m map[string][]string) *CustomTrackersConfig { } } -// MatchTrackers returns a list of names of all trackers that match the given labels. +// MatchTrackers returns a list of names of all trackers and labels that match the given labels. +// a "tracker" label is added to the matched list. func (c *CustomTrackersConfig) MatchTrackers(lbs labels.Labels) []labels.Labels { trackers := make([]labels.Labels, 0) Outer: From fede4cfe8efa306be73fa4703640e3c98ae35c34 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 22 Feb 2024 16:27:44 +0100 Subject: [PATCH 25/31] Rename tracker interface and methods --- pkg/distributor/distributor.go | 2 +- pkg/distributor/validator.go | 2 +- pkg/ingester/instance.go | 2 +- pkg/loghttp/push/custom_trackers.go | 6 +++--- pkg/loghttp/push/push.go | 6 +++--- pkg/loghttp/push/push_test.go | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 2b668585b0cea..4649f6d68942e 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -128,7 +128,7 @@ type Distributor struct { replicationFactor prometheus.Gauge streamShardCount prometheus.Counter - customStreamsTracker push.CustomTracker + customStreamsTracker push.CustomStreamsTracker } // New a distributor creates. diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index caeb91ee3acaf..0e2e9d9d9f574 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -19,7 +19,7 @@ const ( type Validator struct { Limits - customStreamsTracker push.CustomTracker + customStreamsTracker push.CustomStreamsTracker } func NewValidator(l Limits) (*Validator, error) { diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index e8f70ed018436..f97df27f6f326 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -121,7 +121,7 @@ type instance struct { schemaconfig *config.SchemaConfig - customStreamsTracker push.CustomTracker + customStreamsTracker push.CustomStreamsTracker } func newInstance( diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index 9a542c6762be0..d79b21a6245ae 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -6,10 +6,10 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -type CustomTracker interface { +type CustomStreamsTracker interface { - // IngestedBytesAdd records ingested bytes by tenant, retention period and labels. - IngestedBytesAdd(tenant string, retentionPeriod time.Duration, labels labels.Labels, value float64) + // ReceivedBytesAdd records ingested bytes by tenant, retention period and labels. + ReceivedBytesAdd(tenant string, retentionPeriod time.Duration, labels labels.Labels, value float64) // DiscardedBytesAdd records discarded bytes by tenant and labels. DiscardedBytesAdd(tenant string, labels labels.Labels, value float64) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 8421a574a10b8..f64101e935e5a 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -96,7 +96,7 @@ type Stats struct { structuredMetadataBytesCustomTrackers []customTrackerPair } -func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker CustomTracker) (*logproto.PushRequest, error) { +func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker CustomStreamsTracker) (*logproto.PushRequest, error) { req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits) if err != nil { return nil, err @@ -130,12 +130,12 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete if tracker != nil { for _, pair := range pushStats.logLinesBytesCustomTrackers { for retentionPeriod, size := range pair.Bytes { - tracker.IngestedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) + tracker.ReceivedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) } } for _, pair := range pushStats.structuredMetadataBytesCustomTrackers { for retentionPeriod, size := range pair.Bytes { - tracker.IngestedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) + tracker.ReceivedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) } } } diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index 16dbe24763b90..84f401d965035 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -271,7 +271,7 @@ func (t *mockCustomTracker) DiscardedBytesAdd(_ string, labels labels.Labels, va t.discardedBytes[labels.String()] += value } -// IngestedBytesAdd implements CustomTracker. -func (t *mockCustomTracker) IngestedBytesAdd(_ string, _ time.Duration, labels labels.Labels, value float64) { +// ReceivedBytesAdd implements CustomTracker. +func (t *mockCustomTracker) ReceivedBytesAdd(_ string, _ time.Duration, labels labels.Labels, value float64) { t.receivedBytes[labels.String()] += value } From 94986285e7ec0f7ad399c5ed04d0ebb843cb40d6 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 23 Feb 2024 16:02:32 +0100 Subject: [PATCH 26/31] Remove labels filtering and configuration. --- docs/sources/configure/_index.md | 5 - pkg/bloomcompactor/bloomcompactor.go | 2 +- pkg/distributor/distributor.go | 4 +- pkg/distributor/limits.go | 1 - pkg/distributor/validator.go | 27 ++--- pkg/ingester/instance.go | 4 +- pkg/ingester/limiter.go | 6 -- pkg/loghttp/push/custom_trackers.go | 51 +--------- pkg/loghttp/push/custom_trackers_test.go | 25 ----- pkg/loghttp/push/otlp.go | 26 ++--- pkg/loghttp/push/otlp_test.go | 124 ++++++++++------------- pkg/loghttp/push/push.go | 53 ++-------- pkg/loghttp/push/push_test.go | 38 +++---- pkg/logql/rangemapper.go | 2 +- pkg/validation/limits.go | 10 -- 15 files changed, 96 insertions(+), 282 deletions(-) delete mode 100644 pkg/loghttp/push/custom_trackers_test.go diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index f0d61041b4ea1..70891a0448419 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -3234,11 +3234,6 @@ otlp_config: # Configuration for log attributes to store them as Structured Metadata or # drop them altogether [log_attributes: ] - -# Defines a set of custom trackers for ingested bytes. -# Takes a map of tracker names as keys and a list of label names as values. A -# tracker matches a stream if all labels are present. -custom_trackers: ``` ### frontend_worker diff --git a/pkg/bloomcompactor/bloomcompactor.go b/pkg/bloomcompactor/bloomcompactor.go index 85bca48f54f3d..3fcc03bb35d24 100644 --- a/pkg/bloomcompactor/bloomcompactor.go +++ b/pkg/bloomcompactor/bloomcompactor.go @@ -199,7 +199,7 @@ func (c *Compactor) ownsTenant(tenant string) ([]v1.FingerprintBounds, bool, err return nil, false, nil } - // TOOD(owen-d): use .GetTokenRangesForInstance() + // TODO(owen-d): use .GetTokenRangesForInstance() // when it's supported for non zone-aware rings // instead of doing all this manually diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 4649f6d68942e..d49667ed95152 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -428,8 +428,8 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log discardedStreamBytes += len(e.Line) } - for _, matchedLbs := range validationContext.customTrackerConfig.MatchTrackers(lbs) { - d.customStreamsTracker.DiscardedBytesAdd(tenantID, matchedLbs, float64(discardedStreamBytes)) + if d.customStreamsTracker != nil { + d.customStreamsTracker.DiscardedBytesAdd(tenantID, validation.RateLimited, lbs, float64(discardedStreamBytes)) } } } diff --git a/pkg/distributor/limits.go b/pkg/distributor/limits.go index 6d4c8080b49a5..6db6995662dd2 100644 --- a/pkg/distributor/limits.go +++ b/pkg/distributor/limits.go @@ -31,5 +31,4 @@ type Limits interface { MaxStructuredMetadataSize(userID string) int MaxStructuredMetadataCount(userID string) int OTLPConfig(userID string) push.OTLPConfig - CustomTrackersConfig(userID string) *push.CustomTrackersConfig } diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index 0e2e9d9d9f574..6b7eb1a240ae5 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -48,8 +48,6 @@ type validationContext struct { maxStructuredMetadataCount int userID string - - customTrackerConfig *push.CustomTrackersConfig } func (v Validator) getValidationContextForTime(now time.Time, userID string) validationContext { @@ -67,7 +65,6 @@ func (v Validator) getValidationContextForTime(now time.Time, userID string) val allowStructuredMetadata: v.AllowStructuredMetadata(userID), maxStructuredMetadataSize: v.MaxStructuredMetadataSize(userID), maxStructuredMetadataCount: v.MaxStructuredMetadataCount(userID), - customTrackerConfig: v.CustomTrackersConfig(userID), } } @@ -83,9 +80,7 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en validation.DiscardedSamples.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Add(float64(len(entry.Line))) if v.customStreamsTracker != nil { - for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) - } + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.GreaterThanMaxSampleAge, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.GreaterThanMaxSampleAgeErrorMsg, labels, formatedEntryTime, formatedRejectMaxAgeTime) } @@ -95,9 +90,7 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en validation.DiscardedSamples.WithLabelValues(validation.TooFarInFuture, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.TooFarInFuture, ctx.userID).Add(float64(len(entry.Line))) if v.customStreamsTracker != nil { - for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) - } + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.TooFarInFuture, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.TooFarInFutureErrorMsg, labels, formatedEntryTime) } @@ -110,9 +103,7 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en validation.DiscardedSamples.WithLabelValues(validation.LineTooLong, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.LineTooLong, ctx.userID).Add(float64(len(entry.Line))) if v.customStreamsTracker != nil { - for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) - } + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.LineTooLong, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.LineTooLongErrorMsg, maxSize, labels, len(entry.Line)) } @@ -122,9 +113,7 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en validation.DiscardedSamples.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Add(float64(len(entry.Line))) if v.customStreamsTracker != nil { - for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) - } + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.DisallowedStructuredMetadata, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, labels) } @@ -139,9 +128,7 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Add(float64(len(entry.Line))) if v.customStreamsTracker != nil { - for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) - } + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooLarge, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, labels, structuredMetadataSizeBytes, ctx.maxStructuredMetadataSize) } @@ -150,9 +137,7 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Add(float64(len(entry.Line))) if v.customStreamsTracker != nil { - for _, matchedLbs := range ctx.customTrackerConfig.MatchTrackers(labels) { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, matchedLbs, float64(len(entry.Line))) - } + v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooMany, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, labels, structuredMetadataCount, ctx.maxStructuredMetadataCount) } diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index f97df27f6f326..a763c796a4f62 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -300,9 +300,7 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *wal.Recor } validation.DiscardedBytes.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(bytes)) if i.customStreamsTracker != nil { - for _, matchedLbs := range i.limiter.CustumTrackersConfig(i.instanceID).MatchTrackers(labels) { - i.customStreamsTracker.DiscardedBytesAdd(i.instanceID, matchedLbs, float64(bytes)) - } + i.customStreamsTracker.DiscardedBytesAdd(i.instanceID, validation.StreamLimit, labels, float64(bytes)) } return nil, httpgrpc.Errorf(http.StatusTooManyRequests, validation.StreamLimitErrorMsg, i.instanceID) } diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index 599ae417b1474..e48c2a018d277 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -9,7 +9,6 @@ import ( "golang.org/x/time/rate" "github.com/grafana/loki/pkg/distributor/shardstreams" - "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/validation" ) @@ -29,7 +28,6 @@ type Limits interface { MaxGlobalStreamsPerUser(userID string) int PerStreamRateLimit(userID string) validation.RateLimit ShardStreams(userID string) *shardstreams.Config - CustomTrackersConfig(userID string) *push.CustomTrackersConfig } // Limiter implements primitives to get the maximum number of streams @@ -141,10 +139,6 @@ func (l *Limiter) minNonZero(first, second int) int { return first } -func (l *Limiter) CustumTrackersConfig(userID string) *push.CustomTrackersConfig { - return l.limits.CustomTrackersConfig(userID) -} - type RateLimiterStrategy interface { RateLimit(tenant string) validation.RateLimit } diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/custom_trackers.go index d79b21a6245ae..0e59f48fd5fcd 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/custom_trackers.go @@ -9,55 +9,10 @@ import ( type CustomStreamsTracker interface { // ReceivedBytesAdd records ingested bytes by tenant, retention period and labels. + // TODO: pass context ReceivedBytesAdd(tenant string, retentionPeriod time.Duration, labels labels.Labels, value float64) // DiscardedBytesAdd records discarded bytes by tenant and labels. - DiscardedBytesAdd(tenant string, labels labels.Labels, value float64) -} - -// CustomTrackersConfig defines a map of tracker name to tracked label set. -type CustomTrackersConfig struct { - source map[string][]string -} - -var EmptyCustomTrackersConfig = &CustomTrackersConfig{ - source: map[string][]string{}, -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -// CustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. -func (c *CustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - stringMap := map[string][]string{} - err := unmarshal(&stringMap) - if err != nil { - return err - } - *c = *NewCustomTrackersConfig(stringMap) - return nil -} - -func NewCustomTrackersConfig(m map[string][]string) *CustomTrackersConfig { - return &CustomTrackersConfig{ - source: m, - } -} - -// MatchTrackers returns a list of names of all trackers and labels that match the given labels. -// a "tracker" label is added to the matched list. -func (c *CustomTrackersConfig) MatchTrackers(lbs labels.Labels) []labels.Labels { - trackers := make([]labels.Labels, 0) -Outer: - for tracker, requiredLbs := range c.source { - matchedLabels := make(labels.Labels, 0, len(requiredLbs)) - for _, name := range requiredLbs { - if value := lbs.Get(name); value != "" { - matchedLabels = append(matchedLabels, labels.Label{Name: name, Value: value}) - } else { - continue Outer - } - } - matchedLabels = append(matchedLabels, labels.Label{Name: "tracker", Value: tracker}) - trackers = append(trackers, matchedLabels) - } - return trackers + // TODO: pass context + DiscardedBytesAdd(tenant, reason string, labels labels.Labels, value float64) } diff --git a/pkg/loghttp/push/custom_trackers_test.go b/pkg/loghttp/push/custom_trackers_test.go deleted file mode 100644 index 1141071c90cf8..0000000000000 --- a/pkg/loghttp/push/custom_trackers_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package push - -import ( - "testing" - - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" -) - -func Test_Unmarshal(t *testing.T) { - raw := ` -dev: - - region - - zone -integrations: - - job -` - var out CustomTrackersConfig - err := yaml.Unmarshal([]byte(raw), &out) - require.NoError(t, err) - require.Contains(t, out.source, "dev") - require.Contains(t, out.source, "integrations") - require.ElementsMatch(t, out.source["dev"], []string{"region", "zone"}) - require.ElementsMatch(t, out.source["integrations"], []string{"job"}) -} diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go index 5d6c259987362..68ec4895fe32f 100644 --- a/pkg/loghttp/push/otlp.go +++ b/pkg/loghttp/push/otlp.go @@ -43,14 +43,14 @@ func newPushStats() *Stats { } } -func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) { +func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker CustomStreamsTracker) (*logproto.PushRequest, *Stats, error) { stats := newPushStats() otlpLogs, err := extractLogs(r, stats) if err != nil { return nil, nil, err } - req := otlpToLokiPushRequest(otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), limits.CustomTrackersConfig(userID), stats) + req := otlpToLokiPushRequest(otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), tracker, stats) return req, stats, nil } @@ -101,7 +101,7 @@ func extractLogs(r *http.Request, pushStats *Stats) (plog.Logs, error) { return req.Logs(), nil } -func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, customTrackersConfig *CustomTrackersConfig, stats *Stats) *logproto.PushRequest { +func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, tracker CustomStreamsTracker, stats *Stats) *logproto.PushRequest { if ld.LogRecordCount() == 0 { return &logproto.PushRequest{} } @@ -146,17 +146,6 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants lbs := modelLabelsSetToLabelsList(streamLabels) - // Init custom streams tracking - trackedLabels := customTrackersConfig.MatchTrackers(lbs) - stats.structuredMetadataBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) - stats.logLinesBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) - for i, labels := range trackedLabels { - stats.structuredMetadataBytesCustomTrackers[i].Labels = labels - stats.structuredMetadataBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} - stats.logLinesBytesCustomTrackers[i].Labels = labels - stats.logLinesBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} - } - if _, ok := pushRequestsByStream[labelsStr]; !ok { pushRequestsByStream[labelsStr] = logproto.Stream{ Labels: labelsStr, @@ -235,12 +224,13 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants stream.Entries = append(stream.Entries, entry) pushRequestsByStream[labelsStr] = stream - stats.structuredMetadataBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) + metadataSize := int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) + stats.structuredMetadataBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += metadataSize stats.logLinesBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) - for i := range trackedLabels { - stats.logLinesBytesCustomTrackers[i].Bytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) - stats.structuredMetadataBytesCustomTrackers[i].Bytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) + if tracker != nil { + tracker.ReceivedBytesAdd(userID, tenantsRetention.RetentionPeriodFor(userID, lbs), lbs, float64(len(entry.Line))) + tracker.ReceivedBytesAdd(userID, tenantsRetention.RetentionPeriodFor(userID, lbs), lbs, float64(metadataSize)) } stats.numLines++ diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go index 0bdc3f72f5c58..cfa5f0b16ca80 100644 --- a/pkg/loghttp/push/otlp_test.go +++ b/pkg/loghttp/push/otlp_test.go @@ -20,22 +20,21 @@ func TestOTLPToLokiPushRequest(t *testing.T) { now := time.Unix(0, time.Now().UnixNano()) for _, tc := range []struct { - name string - generateLogs func() plog.Logs - expectedPushRequest logproto.PushRequest - expectedStats Stats - otlpConfig OTLPConfig - customTrackersConfig *CustomTrackersConfig + name string + generateLogs func() plog.Logs + expectedPushRequest logproto.PushRequest + expectedStats Stats + otlpConfig OTLPConfig + tracker CustomStreamsTracker }{ { name: "no logs", generateLogs: func() plog.Logs { return plog.NewLogs() }, - expectedPushRequest: logproto.PushRequest{}, - expectedStats: *newPushStats(), - otlpConfig: DefaultOTLPConfig, - customTrackersConfig: EmptyCustomTrackersConfig, + expectedPushRequest: logproto.PushRequest{}, + expectedStats: *newPushStats(), + otlpConfig: DefaultOTLPConfig, }, { name: "resource with no logs", @@ -44,15 +43,13 @@ func TestOTLPToLokiPushRequest(t *testing.T) { ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.name", "service-1") return ld }, - expectedPushRequest: logproto.PushRequest{}, - expectedStats: *newPushStats(), - otlpConfig: DefaultOTLPConfig, - customTrackersConfig: EmptyCustomTrackersConfig, + expectedPushRequest: logproto.PushRequest{}, + expectedStats: *newPushStats(), + otlpConfig: DefaultOTLPConfig, }, { - name: "resource with a log entry", - otlpConfig: DefaultOTLPConfig, - customTrackersConfig: EmptyCustomTrackersConfig, + name: "resource with a log entry", + otlpConfig: DefaultOTLPConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.name", "service-1") @@ -82,16 +79,13 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 0, }, - streamLabelsSize: 21, - mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: []customTrackerPair{}, - structuredMetadataBytesCustomTrackers: []customTrackerPair{}, + streamLabelsSize: 21, + mostRecentEntryTimestamp: now, }, }, { - name: "no resource attributes defined", - otlpConfig: DefaultOTLPConfig, - customTrackersConfig: EmptyCustomTrackersConfig, + name: "no resource attributes defined", + otlpConfig: DefaultOTLPConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -121,16 +115,14 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 0, }, - streamLabelsSize: 27, - mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: []customTrackerPair{}, - structuredMetadataBytesCustomTrackers: []customTrackerPair{}, + streamLabelsSize: 27, + mostRecentEntryTimestamp: now, }, }, { - name: "service.name not defined in resource attributes", - otlpConfig: DefaultOTLPConfig, - customTrackersConfig: NewCustomTrackersConfig(map[string][]string{"foo": {"service_namespace"}}), + name: "service.name not defined in resource attributes", + otlpConfig: DefaultOTLPConfig, + tracker: NewMockTracker(), generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.namespace", "foo") @@ -162,34 +154,36 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 47, mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: []customTrackerPair{ - { - Labels: []labels.Label{ - {Name: "service_namespace", Value: "foo"}, - {Name: "tracker", Value: "foo"}, - }, - Bytes: map[time.Duration]int64{ - time.Hour: 9, + /* + logLinesBytesCustomTrackers: []customTrackerPair{ + { + Labels: []labels.Label{ + {Name: "service_namespace", Value: "foo"}, + {Name: "tracker", Value: "foo"}, + }, + Bytes: map[time.Duration]int64{ + time.Hour: 9, + }, }, }, - }, - structuredMetadataBytesCustomTrackers: []customTrackerPair{ - { - Labels: []labels.Label{ - {Name: "service_namespace", Value: "foo"}, - {Name: "tracker", Value: "foo"}, - }, - Bytes: map[time.Duration]int64{ - time.Hour: 0, + structuredMetadataBytesCustomTrackers: []customTrackerPair{ + { + Labels: []labels.Label{ + {Name: "service_namespace", Value: "foo"}, + {Name: "tracker", Value: "foo"}, + }, + Bytes: map[time.Duration]int64{ + time.Hour: 0, + }, }, }, - }, + */ }, + //expectedTrackedUsaged: }, { - name: "resource attributes and scope attributes stored as structured metadata", - otlpConfig: DefaultOTLPConfig, - customTrackersConfig: EmptyCustomTrackersConfig, + name: "resource attributes and scope attributes stored as structured metadata", + otlpConfig: DefaultOTLPConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -258,16 +252,13 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 37, }, - streamLabelsSize: 21, - mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: []customTrackerPair{}, - structuredMetadataBytesCustomTrackers: []customTrackerPair{}, + streamLabelsSize: 21, + mostRecentEntryTimestamp: now, }, }, { - name: "attributes with nested data", - otlpConfig: DefaultOTLPConfig, - customTrackersConfig: EmptyCustomTrackersConfig, + name: "attributes with nested data", + otlpConfig: DefaultOTLPConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -345,10 +336,8 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 97, }, - streamLabelsSize: 21, - mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: []customTrackerPair{}, - structuredMetadataBytesCustomTrackers: []customTrackerPair{}, + streamLabelsSize: 21, + mostRecentEntryTimestamp: now, }, }, { @@ -390,7 +379,6 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, }, }, - customTrackersConfig: EmptyCustomTrackersConfig, generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty() @@ -491,16 +479,14 @@ func TestOTLPToLokiPushRequest(t *testing.T) { structuredMetadataBytes: map[time.Duration]int64{ time.Hour: 113, }, - streamLabelsSize: 42, - mostRecentEntryTimestamp: now, - logLinesBytesCustomTrackers: []customTrackerPair{}, - structuredMetadataBytesCustomTrackers: []customTrackerPair{}, + streamLabelsSize: 42, + mostRecentEntryTimestamp: now, }, }, } { t.Run(tc.name, func(t *testing.T) { stats := newPushStats() - pushReq := otlpToLokiPushRequest(tc.generateLogs(), "foo", fakeRetention{}, tc.otlpConfig, tc.customTrackersConfig, stats) + pushReq := otlpToLokiPushRequest(tc.generateLogs(), "foo", fakeRetention{}, tc.otlpConfig, tc.tracker, stats) require.Equal(t, tc.expectedPushRequest, *pushReq) require.Equal(t, tc.expectedStats, *stats) }) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index f64101e935e5a..52e1769646eb7 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -61,7 +61,6 @@ type TenantsRetention interface { type Limits interface { OTLPConfig(userID string) OTLPConfig - CustomTrackersConfig(userID string) *CustomTrackersConfig } type EmptyLimits struct{} @@ -70,16 +69,7 @@ func (EmptyLimits) OTLPConfig(string) OTLPConfig { return DefaultOTLPConfig } -func (EmptyLimits) CustomTrackersConfig(string) *CustomTrackersConfig { - return EmptyCustomTrackersConfig -} - -type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) - -type customTrackerPair struct { - Labels labels.Labels - Bytes map[time.Duration]int64 -} +type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker CustomStreamsTracker) (*logproto.PushRequest, *Stats, error) type Stats struct { errs []error @@ -91,13 +81,10 @@ type Stats struct { contentType string contentEncoding string bodySize int64 - - logLinesBytesCustomTrackers []customTrackerPair - structuredMetadataBytesCustomTrackers []customTrackerPair } func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker CustomStreamsTracker) (*logproto.PushRequest, error) { - req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits) + req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits, tracker) if err != nil { return nil, err } @@ -126,20 +113,6 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete structuredMetadataSize += size } - // Process custom trackers - if tracker != nil { - for _, pair := range pushStats.logLinesBytesCustomTrackers { - for retentionPeriod, size := range pair.Bytes { - tracker.ReceivedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) - } - } - for _, pair := range pushStats.structuredMetadataBytesCustomTrackers { - for retentionPeriod, size := range pair.Bytes { - tracker.ReceivedBytesAdd(userID, retentionPeriod, pair.Labels, float64(size)) - } - } - } - // incrementing tenant metrics if we have a tenant. if pushStats.numLines != 0 && userID != "" { linesIngested.WithLabelValues(userID).Add(float64(pushStats.numLines)) @@ -163,7 +136,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete return req, nil } -func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) { +func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, _ Limits, tracker CustomStreamsTracker) (*logproto.PushRequest, *Stats, error) { // Body var body io.Reader // bodySize should always reflect the compressed size of the request body @@ -235,25 +208,13 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe for _, s := range req.Streams { pushStats.streamLabelsSize += int64(len(s.Labels)) - // Init custom trackers - customTrackers := limits.CustomTrackersConfig(userID) var lbs labels.Labels - if tenantsRetention != nil || len(customTrackers.source) > 0 { + if tenantsRetention != nil || tracker != nil { lbs, err = syntax.ParseLabels(s.Labels) if err != nil { return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) } } - trackedLabels := customTrackers.MatchTrackers(lbs) - // TODO: avoid allocation if if cap() <= len(trackerLabels) - pushStats.logLinesBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) - pushStats.structuredMetadataBytesCustomTrackers = make([]customTrackerPair, len(trackedLabels)) - for i, labels := range trackedLabels { - pushStats.logLinesBytesCustomTrackers[i].Labels = labels - pushStats.logLinesBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} - pushStats.structuredMetadataBytesCustomTrackers[i].Labels = labels - pushStats.structuredMetadataBytesCustomTrackers[i].Bytes = map[time.Duration]int64{} - } var retentionPeriod time.Duration if tenantsRetention != nil { @@ -268,9 +229,9 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe pushStats.logLinesBytes[retentionPeriod] += int64(len(e.Line)) pushStats.structuredMetadataBytes[retentionPeriod] += entryLabelsSize - for i := range trackedLabels { - pushStats.logLinesBytesCustomTrackers[i].Bytes[retentionPeriod] += int64(len(e.Line)) - pushStats.structuredMetadataBytesCustomTrackers[i].Bytes[retentionPeriod] += entryLabelsSize + if tracker != nil { + tracker.ReceivedBytesAdd(userID, retentionPeriod, lbs, float64(len(e.Line))) + tracker.ReceivedBytesAdd(userID, retentionPeriod, lbs, float64(entryLabelsSize)) } if e.Timestamp.After(pushStats.mostRecentEntryTimestamp) { diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index 84f401d965035..7593867caffaf 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -206,15 +206,8 @@ func TestParseRequest(t *testing.T) { request.Header.Add("Content-Encoding", test.contentEncoding) } - customTrackersConfig := NewCustomTrackersConfig(test.customTrackers) - limits := &mockLimits{ - customTrackers: customTrackersConfig, - } - tracker := &mockCustomTracker{ - receivedBytes: map[string]float64{}, - discardedBytes: map[string]float64{}, - } - data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest, tracker) + tracker := NewMockTracker() + data, err := ParseRequest(util_log.Logger, "fake", request, nil, nil, ParseLokiRequest, tracker) structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived previousStructuredMetadataBytesReceived += structuredMetadataBytesReceived @@ -247,31 +240,24 @@ func TestParseRequest(t *testing.T) { } } -type mockLimits struct { - customTrackers *CustomTrackersConfig -} - -// CustomTrackersConfig implements Limits. -func (m *mockLimits) CustomTrackersConfig(string) *CustomTrackersConfig { - return m.customTrackers -} - -// OTLPConfig implements Limits. -func (*mockLimits) OTLPConfig(string) OTLPConfig { - return DefaultOTLPConfig -} - -type mockCustomTracker struct { +type MockCustomTracker struct { receivedBytes map[string]float64 discardedBytes map[string]float64 } +func NewMockTracker() *MockCustomTracker { + return &MockCustomTracker{ + receivedBytes: map[string]float64{}, + discardedBytes: map[string]float64{}, + } +} + // DiscardedBytesAdd implements CustomTracker. -func (t *mockCustomTracker) DiscardedBytesAdd(_ string, labels labels.Labels, value float64) { +func (t *MockCustomTracker) DiscardedBytesAdd(_, _ string, labels labels.Labels, value float64) { t.discardedBytes[labels.String()] += value } // ReceivedBytesAdd implements CustomTracker. -func (t *mockCustomTracker) ReceivedBytesAdd(_ string, _ time.Duration, labels labels.Labels, value float64) { +func (t *MockCustomTracker) ReceivedBytesAdd(_ string, _ time.Duration, labels labels.Labels, value float64) { t.receivedBytes[labels.String()] += value } diff --git a/pkg/logql/rangemapper.go b/pkg/logql/rangemapper.go index 14cf76f1475a5..f898e19d2ea1e 100644 --- a/pkg/logql/rangemapper.go +++ b/pkg/logql/rangemapper.go @@ -61,7 +61,7 @@ type RangeMapper struct { splitAlignTs time.Time } -// NewRangeMapperWithSplitAlign is similar to `NewRangeMapper` except it accepts additonal `splitAlign` argument and used to +// NewRangeMapperWithSplitAlign is similar to `NewRangeMapper` except it accepts additional `splitAlign` argument and used to // align the subqueries generated according to that. Look at `rangeSplitAlign` method for more information. func NewRangeMapperWithSplitAlign(interval time.Duration, splitAlign time.Time, metrics *MapperMetrics, stats *MapperStats) (RangeMapper, error) { rm, err := NewRangeMapper(interval, metrics, stats) diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 36e95f682dbf7..ab845380f9682 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -206,8 +206,6 @@ type Limits struct { MaxStructuredMetadataSize flagext.ByteSize `yaml:"max_structured_metadata_size" json:"max_structured_metadata_size" doc:"description=Maximum size accepted for structured metadata per log line."` MaxStructuredMetadataEntriesCount int `yaml:"max_structured_metadata_entries_count" json:"max_structured_metadata_entries_count" doc:"description=Maximum number of structured metadata entries per log line."` OTLPConfig push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"` - - CustomTrackersConfig *push.CustomTrackersConfig `yaml:"custom_trackers" json:"custom_trackers" doc:"description=Defines a set of custom trackers for ingested bytes.\nTakes a map of tracker names as keys and a list of label names as values. A tracker matches a stream if all labels are present."` } type StreamRetention struct { @@ -938,14 +936,6 @@ func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig { return o.getOverridesForUser(userID).OTLPConfig } -func (o *Overrides) CustomTrackersConfig(userID string) *push.CustomTrackersConfig { - if o.getOverridesForUser(userID).CustomTrackersConfig == nil { - return push.EmptyCustomTrackersConfig - } - - return o.getOverridesForUser(userID).CustomTrackersConfig -} - func (o *Overrides) getOverridesForUser(userID string) *Limits { if o.tenantLimits != nil { l := o.tenantLimits.TenantLimits(userID) From ab4f152eae48361ba4d864cf6d021b84ce54924e Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 23 Feb 2024 16:07:17 +0100 Subject: [PATCH 27/31] Rename tracker --- pkg/distributor/distributor.go | 2 +- pkg/distributor/validator.go | 2 +- pkg/ingester/instance.go | 2 +- pkg/loghttp/push/otlp.go | 4 ++-- pkg/loghttp/push/otlp_test.go | 2 +- pkg/loghttp/push/push.go | 6 +++--- pkg/loghttp/push/{custom_trackers.go => usage_tracker.go} | 4 +--- 7 files changed, 10 insertions(+), 12 deletions(-) rename pkg/loghttp/push/{custom_trackers.go => usage_tracker.go} (83%) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index d49667ed95152..430d5a54072a1 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -128,7 +128,7 @@ type Distributor struct { replicationFactor prometheus.Gauge streamShardCount prometheus.Counter - customStreamsTracker push.CustomStreamsTracker + customStreamsTracker push.UsageTracker } // New a distributor creates. diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index 6b7eb1a240ae5..869d4dc72e3ca 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -19,7 +19,7 @@ const ( type Validator struct { Limits - customStreamsTracker push.CustomStreamsTracker + customStreamsTracker push.UsageTracker } func NewValidator(l Limits) (*Validator, error) { diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index a763c796a4f62..c7953ea1aba1e 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -121,7 +121,7 @@ type instance struct { schemaconfig *config.SchemaConfig - customStreamsTracker push.CustomStreamsTracker + customStreamsTracker push.UsageTracker } func newInstance( diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go index 68ec4895fe32f..cb73f6db59eec 100644 --- a/pkg/loghttp/push/otlp.go +++ b/pkg/loghttp/push/otlp.go @@ -43,7 +43,7 @@ func newPushStats() *Stats { } } -func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker CustomStreamsTracker) (*logproto.PushRequest, *Stats, error) { +func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { stats := newPushStats() otlpLogs, err := extractLogs(r, stats) if err != nil { @@ -101,7 +101,7 @@ func extractLogs(r *http.Request, pushStats *Stats) (plog.Logs, error) { return req.Logs(), nil } -func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, tracker CustomStreamsTracker, stats *Stats) *logproto.PushRequest { +func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, tracker UsageTracker, stats *Stats) *logproto.PushRequest { if ld.LogRecordCount() == 0 { return &logproto.PushRequest{} } diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go index cfa5f0b16ca80..593ac380e6696 100644 --- a/pkg/loghttp/push/otlp_test.go +++ b/pkg/loghttp/push/otlp_test.go @@ -25,7 +25,7 @@ func TestOTLPToLokiPushRequest(t *testing.T) { expectedPushRequest logproto.PushRequest expectedStats Stats otlpConfig OTLPConfig - tracker CustomStreamsTracker + tracker UsageTracker }{ { name: "no logs", diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 52e1769646eb7..012a70386bd76 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -69,7 +69,7 @@ func (EmptyLimits) OTLPConfig(string) OTLPConfig { return DefaultOTLPConfig } -type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker CustomStreamsTracker) (*logproto.PushRequest, *Stats, error) +type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) type Stats struct { errs []error @@ -83,7 +83,7 @@ type Stats struct { bodySize int64 } -func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker CustomStreamsTracker) (*logproto.PushRequest, error) { +func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker UsageTracker) (*logproto.PushRequest, error) { req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits, tracker) if err != nil { return nil, err @@ -136,7 +136,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete return req, nil } -func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, _ Limits, tracker CustomStreamsTracker) (*logproto.PushRequest, *Stats, error) { +func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, _ Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { // Body var body io.Reader // bodySize should always reflect the compressed size of the request body diff --git a/pkg/loghttp/push/custom_trackers.go b/pkg/loghttp/push/usage_tracker.go similarity index 83% rename from pkg/loghttp/push/custom_trackers.go rename to pkg/loghttp/push/usage_tracker.go index 0e59f48fd5fcd..ab84da5c6acc3 100644 --- a/pkg/loghttp/push/custom_trackers.go +++ b/pkg/loghttp/push/usage_tracker.go @@ -6,13 +6,11 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -type CustomStreamsTracker interface { +type UsageTracker interface { // ReceivedBytesAdd records ingested bytes by tenant, retention period and labels. - // TODO: pass context ReceivedBytesAdd(tenant string, retentionPeriod time.Duration, labels labels.Labels, value float64) // DiscardedBytesAdd records discarded bytes by tenant and labels. - // TODO: pass context DiscardedBytesAdd(tenant, reason string, labels labels.Labels, value float64) } From 9df9532478bc07873920063254fc3c72cf0988e6 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 23 Feb 2024 16:07:43 +0100 Subject: [PATCH 28/31] Correct changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1181190339292..fb09c05dded79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ##### Enhancements -* [11840](https://github.com/grafana/loki/pull/11840) **jeschkies**: Allow custom trackers for ingested and discarded bytes metric. +* [11840](https://github.com/grafana/loki/pull/11840) **jeschkies**: Allow custom usage trackers for ingested and discarded bytes metric. * [11814](https://github.com/grafana/loki/pull/11814) **kavirajk**: feat: Support split align and caching for instant metric query results * [11851](https://github.com/grafana/loki/pull/11851) **elcomtik**: Helm: Allow the definition of resources for GrafanaAgent pods. * [11819](https://github.com/grafana/loki/pull/11819) **jburnham**: Ruler: Add the ability to disable the `X-Scope-OrgId` tenant identification header in remote write requests. From c6a7cb4dcd571f4af9485f886529961418425b99 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 23 Feb 2024 16:13:00 +0100 Subject: [PATCH 29/31] Inject usage tracker --- pkg/distributor/distributor.go | 12 +++++++----- pkg/distributor/distributor_test.go | 8 ++++---- pkg/distributor/http.go | 2 +- pkg/distributor/validator.go | 30 ++++++++++++++--------------- pkg/distributor/validator_test.go | 4 ++-- pkg/loki/loki.go | 3 +++ pkg/loki/modules.go | 1 + 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 430d5a54072a1..d7c28ae8b74b4 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -128,7 +128,7 @@ type Distributor struct { replicationFactor prometheus.Gauge streamShardCount prometheus.Counter - customStreamsTracker push.UsageTracker + usageTracker push.UsageTracker } // New a distributor creates. @@ -141,6 +141,7 @@ func New( registerer prometheus.Registerer, metricsNamespace string, tee Tee, + usageTracker push.UsageTracker, logger log.Logger, ) (*Distributor, error) { factory := cfg.factory @@ -156,7 +157,7 @@ func New( return client.New(internalCfg, addr) } - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, usageTracker) if err != nil { return nil, err } @@ -188,6 +189,7 @@ func New( healthyInstancesCount: atomic.NewUint32(0), rateLimitStrat: rateLimitStrat, tee: tee, + usageTracker: usageTracker, ingesterAppends: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_ingester_appends_total", @@ -416,7 +418,7 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log validation.DiscardedSamples.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineCount)) validation.DiscardedBytes.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineSize)) - if d.customStreamsTracker != nil { + if d.usageTracker != nil { for _, stream := range req.Streams { lbs, _, _, err := d.parseStreamLabels(validationContext, stream.Labels, &stream) if err != nil { @@ -428,8 +430,8 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log discardedStreamBytes += len(e.Line) } - if d.customStreamsTracker != nil { - d.customStreamsTracker.DiscardedBytesAdd(tenantID, validation.RateLimited, lbs, float64(discardedStreamBytes)) + if d.usageTracker != nil { + d.usageTracker.DiscardedBytesAdd(tenantID, validation.RateLimited, lbs, float64(discardedStreamBytes)) } } } diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 49479648063fd..04747ffb72334 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -612,7 +612,7 @@ func TestStreamShard(t *testing.T) { overrides, err := validation.NewOverrides(*distributorLimits, nil) require.NoError(t, err) - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, nil) require.NoError(t, err) d := Distributor{ @@ -656,7 +656,7 @@ func TestStreamShardAcrossCalls(t *testing.T) { overrides, err := validation.NewOverrides(*distributorLimits, nil) require.NoError(t, err) - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, nil) require.NoError(t, err) t.Run("it generates 4 shards across 2 calls when calculated shards = 2 * entries per call", func(t *testing.T) { @@ -721,7 +721,7 @@ func BenchmarkShardStream(b *testing.B) { overrides, err := validation.NewOverrides(*distributorLimits, nil) require.NoError(b, err) - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, nil) require.NoError(b, err) distributorBuilder := func(shards int) *Distributor { @@ -1159,7 +1159,7 @@ func prepare(t *testing.T, numDistributors, numIngesters int, limits *validation overrides, err := validation.NewOverrides(*limits, nil) require.NoError(t, err) - d, err := New(distributorConfig, clientConfig, runtime.DefaultTenantConfigs(), ingestersRing, overrides, prometheus.NewPedanticRegistry(), constants.Loki, nil, log.NewNopLogger()) + d, err := New(distributorConfig, clientConfig, runtime.DefaultTenantConfigs(), ingestersRing, overrides, prometheus.NewPedanticRegistry(), constants.Loki, nil, nil, log.NewNopLogger()) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), d)) distributors[i] = d diff --git a/pkg/distributor/http.go b/pkg/distributor/http.go index 6b447b07ecf59..d2582f027f9b9 100644 --- a/pkg/distributor/http.go +++ b/pkg/distributor/http.go @@ -34,7 +34,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe http.Error(w, err.Error(), http.StatusBadRequest) return } - req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser, d.customStreamsTracker) + req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser, d.usageTracker) if err != nil { if d.tenantConfigs.LogPushRequest(tenantID) { level.Debug(logger).Log( diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index 869d4dc72e3ca..f1f2e4acb0ea6 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -19,14 +19,14 @@ const ( type Validator struct { Limits - customStreamsTracker push.UsageTracker + usageTracker push.UsageTracker } -func NewValidator(l Limits) (*Validator, error) { +func NewValidator(l Limits, t push.UsageTracker) (*Validator, error) { if l == nil { return nil, errors.New("nil Limits") } - return &Validator{l, nil}, nil + return &Validator{l, t}, nil } type validationContext struct { @@ -79,8 +79,8 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en formatedRejectMaxAgeTime := time.Unix(0, ctx.rejectOldSampleMaxAge).Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Add(float64(len(entry.Line))) - if v.customStreamsTracker != nil { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.GreaterThanMaxSampleAge, labels, float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.GreaterThanMaxSampleAge, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.GreaterThanMaxSampleAgeErrorMsg, labels, formatedEntryTime, formatedRejectMaxAgeTime) } @@ -89,8 +89,8 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en formatedEntryTime := entry.Timestamp.Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.TooFarInFuture, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.TooFarInFuture, ctx.userID).Add(float64(len(entry.Line))) - if v.customStreamsTracker != nil { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.TooFarInFuture, labels, float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.TooFarInFuture, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.TooFarInFutureErrorMsg, labels, formatedEntryTime) } @@ -102,8 +102,8 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en // for parity. validation.DiscardedSamples.WithLabelValues(validation.LineTooLong, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.LineTooLong, ctx.userID).Add(float64(len(entry.Line))) - if v.customStreamsTracker != nil { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.LineTooLong, labels, float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.LineTooLong, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.LineTooLongErrorMsg, maxSize, labels, len(entry.Line)) } @@ -112,8 +112,8 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en if !ctx.allowStructuredMetadata { validation.DiscardedSamples.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Add(float64(len(entry.Line))) - if v.customStreamsTracker != nil { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.DisallowedStructuredMetadata, labels, float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.DisallowedStructuredMetadata, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, labels) } @@ -127,8 +127,8 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en if maxSize := ctx.maxStructuredMetadataSize; maxSize != 0 && structuredMetadataSizeBytes > maxSize { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Add(float64(len(entry.Line))) - if v.customStreamsTracker != nil { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooLarge, labels, float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooLarge, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, labels, structuredMetadataSizeBytes, ctx.maxStructuredMetadataSize) } @@ -136,8 +136,8 @@ func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, en if maxCount := ctx.maxStructuredMetadataCount; maxCount != 0 && structuredMetadataCount > maxCount { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Add(float64(len(entry.Line))) - if v.customStreamsTracker != nil { - v.customStreamsTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooMany, labels, float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooMany, labels, float64(len(entry.Line))) } return fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, labels, structuredMetadataCount, ctx.maxStructuredMetadataCount) } diff --git a/pkg/distributor/validator_test.go b/pkg/distributor/validator_test.go index 5656f24de1855..0c37065e30563 100644 --- a/pkg/distributor/validator_test.go +++ b/pkg/distributor/validator_test.go @@ -127,7 +127,7 @@ func TestValidator_ValidateEntry(t *testing.T) { flagext.DefaultValues(l) o, err := validation.NewOverrides(*l, tt.overrides) assert.NoError(t, err) - v, err := NewValidator(o) + v, err := NewValidator(o, nil) assert.NoError(t, err) err = v.ValidateEntry(v.getValidationContextForTime(testTime, tt.userID), testStreamLabels, tt.entry) @@ -225,7 +225,7 @@ func TestValidator_ValidateLabels(t *testing.T) { flagext.DefaultValues(l) o, err := validation.NewOverrides(*l, tt.overrides) assert.NoError(t, err) - v, err := NewValidator(o) + v, err := NewValidator(o, nil) assert.NoError(t, err) err = v.ValidateLabels(v.getValidationContextForTime(testTime, tt.userID), mustParseLabels(tt.labels), logproto.Stream{Labels: tt.labels}) diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 75401decb8fc0..c144ada36b4ed 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -38,6 +38,7 @@ import ( "github.com/grafana/loki/pkg/distributor" "github.com/grafana/loki/pkg/ingester" ingester_client "github.com/grafana/loki/pkg/ingester/client" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/loki/common" "github.com/grafana/loki/pkg/lokifrontend" "github.com/grafana/loki/pkg/lokifrontend/frontend/transport" @@ -330,6 +331,8 @@ type Loki struct { Codec Codec Metrics *server.Metrics + + UsageTracker push.UsageTracker } // New makes a new Loki. diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 9d5a614dc5796..bf4f8812f7788 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -320,6 +320,7 @@ func (t *Loki) initDistributor() (services.Service, error) { prometheus.DefaultRegisterer, t.Cfg.MetricsNamespace, t.Tee, + t.UsageTracker, logger, ) if err != nil { From d07ac8548a954996d85305de46be3a0fd68eb4e9 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 23 Feb 2024 16:20:23 +0100 Subject: [PATCH 30/31] Do not presist label string --- pkg/distributor/distributor.go | 12 +++++------- pkg/logql/syntax/parser.go | 1 - 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index d7c28ae8b74b4..53ff20ed9274e 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -708,15 +708,14 @@ func (d *Distributor) sendStreamsErr(ctx context.Context, ingester ring.Instance } type labelData struct { - ls labels.Labels - labels string - hash uint64 + ls labels.Labels + hash uint64 } func (d *Distributor) parseStreamLabels(vContext validationContext, key string, stream *logproto.Stream) (labels.Labels, string, uint64, error) { if val, ok := d.labelCache.Get(key); ok { labelVal := val.(labelData) - return labelVal.ls, labelVal.labels, labelVal.hash, nil + return labelVal.ls, labelVal.ls.String(), labelVal.hash, nil } ls, err := syntax.ParseLabels(key) @@ -728,11 +727,10 @@ func (d *Distributor) parseStreamLabels(vContext validationContext, key string, return nil, "", 0, err } - lsVal := ls.String() lsHash := ls.Hash() - d.labelCache.Add(key, labelData{ls, lsVal, lsHash}) - return ls, lsVal, lsHash, nil + d.labelCache.Add(key, labelData{ls, lsHash}) + return ls, ls.String(), lsHash, nil } // shardCountFor returns the right number of shards to be used by the given stream. diff --git a/pkg/logql/syntax/parser.go b/pkg/logql/syntax/parser.go index 0ad6a304da284..79213049f376c 100644 --- a/pkg/logql/syntax/parser.go +++ b/pkg/logql/syntax/parser.go @@ -252,7 +252,6 @@ func ParseLogSelector(input string, validate bool) (LogSelectorExpr, error) { // ParseLabels parses labels from a string using logql parser. func ParseLabels(lbs string) (labels.Labels, error) { - // TODO: I wonder if we should use memoization here. There's a cache in ls, err := promql_parser.ParseMetric(lbs) if err != nil { return nil, err From c01afd7b48c96321d1bcee4d505fb437c08c37e7 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Fri, 23 Feb 2024 16:27:13 +0100 Subject: [PATCH 31/31] Correct push test --- pkg/loghttp/push/push_test.go | 96 ++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index 7593867caffaf..ec4fd8c8f818a 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -56,8 +56,7 @@ func TestParseRequest(t *testing.T) { expectedStructuredMetadataBytes int expectedBytes int expectedLines int - customTrackers map[string][]string - expectedBytesCustomTracker map[string]float64 + expectedBytesUsageTracker map[string]float64 }{ { path: `/loki/api/v1/push`, @@ -72,23 +71,23 @@ func TestParseRequest(t *testing.T) { valid: false, }, { - path: `/loki/api/v1/push`, - body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, - contentType: `application/json`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, + contentType: `application/json`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { - path: `/loki/api/v1/push`, - body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, - contentType: `application/json`, - contentEncoding: ``, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, - customTrackers: map[string][]string{"t": {"foo"}}, - expectedBytesCustomTracker: map[string]float64{`{foo="bar2", tracker="t"}`: float64(len("fizzbuss"))}, + path: `/loki/api/v1/push`, + body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, + contentType: `application/json`, + contentEncoding: ``, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { path: `/loki/api/v1/push`, @@ -98,22 +97,24 @@ func TestParseRequest(t *testing.T) { valid: false, }, { - path: `/loki/api/v1/push`, - body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json`, - contentEncoding: `gzip`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json`, + contentEncoding: `gzip`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { - path: `/loki/api/v1/push`, - body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json`, - contentEncoding: `deflate`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json`, + contentEncoding: `deflate`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { path: `/loki/api/v1/push`, @@ -123,22 +124,24 @@ func TestParseRequest(t *testing.T) { valid: false, }, { - path: `/loki/api/v1/push`, - body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json; charset=utf-8`, - contentEncoding: `gzip`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json; charset=utf-8`, + contentEncoding: `gzip`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { - path: `/loki/api/v1/push`, - body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json; charset=utf-8`, - contentEncoding: `deflate`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json; charset=utf-8`, + contentEncoding: `deflate`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { path: `/loki/api/v1/push`, @@ -191,6 +194,7 @@ func TestParseRequest(t *testing.T) { expectedStructuredMetadataBytes: 2*len("a") + 2*len("b"), expectedBytes: len("fizzbuzz") + 2*len("a") + 2*len("b"), expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuzz") + 2*len("a") + 2*len("b"))}, }, } { t.Run(fmt.Sprintf("test %d", index), func(t *testing.T) { @@ -225,7 +229,7 @@ func TestParseRequest(t *testing.T) { require.Equal(t, float64(test.expectedStructuredMetadataBytes), testutil.ToFloat64(structuredMetadataBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedBytes), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedLines), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) - require.InDeltaMapValues(t, test.expectedBytesCustomTracker, tracker.receivedBytes, 0.0) + require.InDeltaMapValuesf(t, test.expectedBytesUsageTracker, tracker.receivedBytes, 0.0, "%s != %s", test.expectedBytesUsageTracker, tracker.receivedBytes) } else { assert.Errorf(t, err, "Should give error for %d", index) assert.Nil(t, data, "Should not give data for %d", index)