-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(api): Support Parquet as a query response format. (#15408)
**What this PR does / why we need it**: This changes adds support for the header `Accept: application/vnd.apacher.parquet`. If this header is set the response for metric and stream queries will response with a Parquet file. A metrics response has the columns `timestamp`, `labels` and `value`. A stream response has the columns `timestamp`, `labels` and `line`. Co-authored-by: Christian Haudum <christian.haudum@gmail.com>
- Loading branch information
Showing
318 changed files
with
309,229 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| package queryrange | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "io" | ||
| "net/http" | ||
|
|
||
| "github.com/opentracing/opentracing-go" | ||
| "github.com/parquet-go/parquet-go" | ||
| "github.com/prometheus/prometheus/promql/parser" | ||
|
|
||
| serverutil "github.com/grafana/loki/v3/pkg/util/server" | ||
|
|
||
| "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase" | ||
| ) | ||
|
|
||
| func encodeResponseParquet(ctx context.Context, res queryrangebase.Response) (*http.Response, error) { | ||
| sp, _ := opentracing.StartSpanFromContext(ctx, "codec.EncodeResponse") | ||
| defer sp.Finish() | ||
|
|
||
| var buf bytes.Buffer | ||
|
|
||
| err := encodeResponseParquetTo(ctx, res, &buf) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| resp := http.Response{ | ||
| Header: http.Header{ | ||
| "Content-Type": []string{ParquetType}, | ||
| }, | ||
| Body: io.NopCloser(&buf), | ||
| StatusCode: http.StatusOK, | ||
| } | ||
| return &resp, nil | ||
| } | ||
|
|
||
| func encodeResponseParquetTo(_ context.Context, res queryrangebase.Response, w io.Writer) error { | ||
| switch response := res.(type) { | ||
| case *LokiPromResponse: | ||
| return encodeMetricsParquetTo(response, w) | ||
| case *LokiResponse: | ||
| return encodeLogsParquetTo(response, w) | ||
| default: | ||
| return serverutil.UserError("request does not support Parquet responses") | ||
| } | ||
| } | ||
|
|
||
| type MetricRowType struct { | ||
| Timestamp int64 `parquet:"timestamp,timestamp(millisecond),delta"` | ||
| Labels map[string]string `parquet:"labels"` | ||
| Value float64 `parquet:"value"` | ||
| } | ||
|
|
||
| type LogStreamRowType struct { | ||
| Timestamp int64 `parquet:"timestamp,timestamp(nanosecond),delta"` | ||
| Labels map[string]string `parquet:"labels"` | ||
| Line string `parquet:"line,lz4"` | ||
| } | ||
|
|
||
| func encodeMetricsParquetTo(response *LokiPromResponse, w io.Writer) error { | ||
| schema := parquet.SchemaOf(new(MetricRowType)) | ||
| writer := parquet.NewGenericWriter[MetricRowType](w, schema) | ||
|
|
||
| for _, stream := range response.Response.Data.Result { | ||
| lbls := make(map[string]string) | ||
| for _, keyValue := range stream.Labels { | ||
| lbls[keyValue.Name] = keyValue.Value | ||
| } | ||
| for _, sample := range stream.Samples { | ||
| row := MetricRowType{ | ||
| Timestamp: sample.TimestampMs, | ||
| Labels: lbls, | ||
| Value: sample.Value, | ||
| } | ||
| if _, err := writer.Write([]MetricRowType{row}); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return writer.Close() | ||
| } | ||
|
|
||
| func encodeLogsParquetTo(response *LokiResponse, w io.Writer) error { | ||
| schema := parquet.SchemaOf(new(LogStreamRowType)) | ||
| writer := parquet.NewGenericWriter[LogStreamRowType](w, schema) | ||
|
|
||
| for _, stream := range response.Data.Result { | ||
| lbls, err := parser.ParseMetric(stream.Labels) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| lblsMap := make(map[string]string) | ||
| for _, keyValue := range lbls { | ||
| lblsMap[keyValue.Name] = keyValue.Value | ||
| } | ||
|
|
||
| for _, entry := range stream.Entries { | ||
| row := LogStreamRowType{ | ||
| Timestamp: entry.Timestamp.UnixNano(), | ||
| Labels: lblsMap, | ||
| Line: entry.Line, | ||
| } | ||
| if _, err := writer.Write([]LogStreamRowType{row}); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return writer.Close() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| package queryrange | ||
|
|
||
| import ( | ||
| "os" | ||
| "testing" | ||
|
|
||
| "github.com/parquet-go/parquet-go" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/grafana/loki/v3/pkg/loghttp" | ||
| "github.com/grafana/loki/v3/pkg/logproto" | ||
| "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase" | ||
| ) | ||
|
|
||
| func TestEncodeMetricsParquet(t *testing.T) { | ||
| resp := &LokiPromResponse{ | ||
| Response: &queryrangebase.PrometheusResponse{ | ||
| Status: loghttp.QueryStatusSuccess, | ||
| Data: queryrangebase.PrometheusData{ | ||
| ResultType: loghttp.ResultTypeMatrix, | ||
| Result: sampleStreams, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| f, err := os.CreateTemp("", "metrics-*.parquet") | ||
| defer f.Close() // nolint:staticcheck | ||
|
|
||
| require.NoError(t, err) | ||
| err = encodeMetricsParquetTo(resp, f) | ||
| require.NoError(t, err) | ||
|
|
||
| rows, err := parquet.ReadFile[MetricRowType](f.Name()) | ||
| require.NoError(t, err) | ||
|
|
||
| require.Len(t, rows, 3) | ||
| } | ||
|
|
||
| func TestEncodeLogsParquet(t *testing.T) { | ||
| resp := &LokiResponse{ | ||
| Status: loghttp.QueryStatusSuccess, | ||
| Direction: logproto.FORWARD, | ||
| Limit: 100, | ||
| Version: uint32(loghttp.VersionV1), | ||
| Data: LokiData{ | ||
| ResultType: loghttp.ResultTypeStream, | ||
| Result: logStreams, | ||
| }, | ||
| } | ||
|
|
||
| f, err := os.CreateTemp("", "logs-*.parquet") | ||
| defer f.Close() // nolint:staticcheck | ||
|
|
||
| require.NoError(t, err) | ||
| err = encodeLogsParquetTo(resp, f) | ||
| require.NoError(t, err) | ||
|
|
||
| rows, err := parquet.ReadFile[LogStreamRowType](f.Name()) | ||
| require.NoError(t, err) | ||
|
|
||
| require.Len(t, rows, 3) | ||
| } |
Empty file.
Oops, something went wrong.