From bb1edc0cab538232e8c02d883d05e098abd78c97 Mon Sep 17 00:00:00 2001 From: Raj Date: Wed, 27 Nov 2024 20:19:01 +0530 Subject: [PATCH] feat: bring in a copy of awsfirehosereceiver from opentelemetry-collector-contrib --- receiver/signozawsfirehosereceiver/Makefile | 1 + receiver/signozawsfirehosereceiver/README.md | 88 ++++++ receiver/signozawsfirehosereceiver/config.go | 39 +++ .../signozawsfirehosereceiver/config_test.go | 57 ++++ receiver/signozawsfirehosereceiver/doc.go | 16 + receiver/signozawsfirehosereceiver/factory.go | 104 +++++++ .../signozawsfirehosereceiver/factory_test.go | 50 +++ .../generated_component_test.go | 76 +++++ .../generated_package_test.go | 13 + receiver/signozawsfirehosereceiver/go.mod | 83 +++++ receiver/signozawsfirehosereceiver/go.sum | 178 +++++++++++ .../internal/metadata/generated_status.go | 17 + .../cwlog/compression/compression.go | 48 +++ .../internal/unmarshaler/cwlog/cwlog.go | 17 + .../internal/unmarshaler/cwlog/logsbuilder.go | 50 +++ .../cwlog/testdata/invalid_records | 4 + .../cwlog/testdata/multiple_records | 2 + .../cwlog/testdata/multiple_resources | 6 + .../unmarshaler/cwlog/testdata/single_record | 1 + .../cwlog/testdata/some_invalid_records | 3 + .../internal/unmarshaler/cwlog/unmarshaler.go | 104 +++++++ .../unmarshaler/cwlog/unmarshaler_test.go | 83 +++++ .../unmarshaler/cwmetricstream/cwmetric.go | 47 +++ .../cwmetricstream/metricsbuilder.go | 160 ++++++++++ .../cwmetricstream/metricsbuilder_test.go | 212 +++++++++++++ .../cwmetricstream/package_test.go | 14 + .../cwmetricstream/testdata/invalid_records | 4 + .../cwmetricstream/testdata/multiple_records | 127 ++++++++ .../cwmetricstream/testdata/single_record | 1 + .../testdata/some_invalid_records | 92 ++++++ .../unmarshaler/cwmetricstream/unmarshaler.go | 99 ++++++ .../cwmetricstream/unmarshaler_test.go | 84 +++++ .../otlpmetricstream/unmarshaler.go | 71 +++++ .../otlpmetricstream/unmarshaler_test.go | 125 ++++++++ .../internal/unmarshaler/unmarshaler.go | 27 ++ .../unmarshalertest/nop_logs_unmarshaler.go | 47 +++ .../nop_logs_unmarshaler_test.go | 41 +++ .../nop_metrics_unmarshaler.go | 49 +++ .../nop_metrics_unmarshaler_test.go | 41 +++ .../unmarshalertest/package_test.go | 14 + .../logs_receiver.go | 86 +++++ .../logs_receiver_test.go | 121 ++++++++ .../signozawsfirehosereceiver/metadata.yaml | 12 + .../metrics_receiver.go | 88 ++++++ .../metrics_receiver_test.go | 122 ++++++++ .../signozawsfirehosereceiver/receiver.go | 293 ++++++++++++++++++ .../receiver_test.go | 252 +++++++++++++++ .../testdata/cwlogs_config.yaml | 7 + .../testdata/cwmetrics_config.yaml | 7 + .../testdata/invalid_config.yaml | 3 + .../testdata/otlp_v1_config.yaml | 7 + 51 files changed, 3293 insertions(+) create mode 100644 receiver/signozawsfirehosereceiver/Makefile create mode 100644 receiver/signozawsfirehosereceiver/README.md create mode 100644 receiver/signozawsfirehosereceiver/config.go create mode 100644 receiver/signozawsfirehosereceiver/config_test.go create mode 100644 receiver/signozawsfirehosereceiver/doc.go create mode 100644 receiver/signozawsfirehosereceiver/factory.go create mode 100644 receiver/signozawsfirehosereceiver/factory_test.go create mode 100644 receiver/signozawsfirehosereceiver/generated_component_test.go create mode 100644 receiver/signozawsfirehosereceiver/generated_package_test.go create mode 100644 receiver/signozawsfirehosereceiver/go.mod create mode 100644 receiver/signozawsfirehosereceiver/go.sum create mode 100644 receiver/signozawsfirehosereceiver/internal/metadata/generated_status.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/compression/compression.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/cwlog.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/logsbuilder.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/invalid_records create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_records create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_resources create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/single_record create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/some_invalid_records create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler_test.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/cwmetric.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder_test.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/package_test.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/invalid_records create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/multiple_records create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/single_record create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/some_invalid_records create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler_test.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler_test.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshaler.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler_test.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler_test.go create mode 100644 receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/package_test.go create mode 100644 receiver/signozawsfirehosereceiver/logs_receiver.go create mode 100644 receiver/signozawsfirehosereceiver/logs_receiver_test.go create mode 100644 receiver/signozawsfirehosereceiver/metadata.yaml create mode 100644 receiver/signozawsfirehosereceiver/metrics_receiver.go create mode 100644 receiver/signozawsfirehosereceiver/metrics_receiver_test.go create mode 100644 receiver/signozawsfirehosereceiver/receiver.go create mode 100644 receiver/signozawsfirehosereceiver/receiver_test.go create mode 100644 receiver/signozawsfirehosereceiver/testdata/cwlogs_config.yaml create mode 100644 receiver/signozawsfirehosereceiver/testdata/cwmetrics_config.yaml create mode 100644 receiver/signozawsfirehosereceiver/testdata/invalid_config.yaml create mode 100644 receiver/signozawsfirehosereceiver/testdata/otlp_v1_config.yaml diff --git a/receiver/signozawsfirehosereceiver/Makefile b/receiver/signozawsfirehosereceiver/Makefile new file mode 100644 index 00000000..ded7a360 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/receiver/signozawsfirehosereceiver/README.md b/receiver/signozawsfirehosereceiver/README.md new file mode 100644 index 00000000..0d6d3879 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/README.md @@ -0,0 +1,88 @@ +# AWS Kinesis Data Firehose Receiver + + +| Status | | +| ------------- |-----------| +| Stability | [alpha]: metrics, logs | +| Distributions | [contrib] | +| Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fawsfirehose%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fawsfirehose) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fawsfirehose%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fawsfirehose) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@Aneurysm9](https://www.github.com/Aneurysm9) | + +[alpha]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#alpha +[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib + + +Receiver for ingesting AWS Kinesis Data Firehose delivery stream messages and parsing the records received based on the configured record type. + +## Configuration + +Example: + +```yaml +receivers: + awsfirehose: + endpoint: 0.0.0.0:4433 + record_type: cwmetrics + access_key: "some_access_key" + tls: + cert_file: server.crt + key_file: server.key +``` +The configuration includes the Opentelemetry collector's server [confighttp](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/confighttp#server-configuration), +which allows for a variety of settings. Only the most relevant ones will be discussed here, but all are available. +The AWS Kinesis Data Firehose Delivery Streams currently only support HTTPS endpoints using port 443. This can be potentially circumvented +using a Load Balancer. + +### endpoint: +The address:port to bind the listener to. + +default: `localhost:4433` + +You can temporarily disable the `component.UseLocalHostAsDefaultHost` feature gate to change this to `0.0.0.0:4433`. This feature gate will be removed in a future release. + +### tls: +See [documentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md#server-configuration) for more details. + +A `cert_file` and `key_file` are required. + +### record_type: +The type of record being received from the delivery stream. Each unmarshaler handles a specific type, so the field allows the receiver to use the correct one. + +default: `cwmetrics` + +See the [Record Types](#record-types) section for all available options. + +### access_key (Optional): +The access key to be checked on each request received. This can be set when creating or updating the delivery stream. +See [documentation](https://docs.aws.amazon.com/firehose/latest/dev/create-destination.html#create-destination-http) for details. + +## Record Types + +### cwmetrics +The record type for the CloudWatch metric stream. Expects the format for the records to be JSON. +See [documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Metric-Streams.html) for details. + +### cwlogs +The record type for the CloudWatch log stream. Expects the format for the records to be JSON. +For example: + +```json +{ + "messageType": "DATA_MESSAGE", + "owner": "111122223333", + "logGroup": "my-log-group", + "logStream": "my-log-stream", + "subscriptionFilters": ["my-subscription-filter"], + "logEvents": [ + { + "id": "123", + "timestamp": 1725544035523, + "message": "My log message." + } + ] +} +``` + +### otlp_v1 +The OTLP v1 format as produced by CloudWatch metric streams. +See [documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats-opentelemetry-100.html) for details. diff --git a/receiver/signozawsfirehosereceiver/config.go b/receiver/signozawsfirehosereceiver/config.go new file mode 100644 index 00000000..784e0af2 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/config.go @@ -0,0 +1,39 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver" + +import ( + "errors" + + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configopaque" +) + +type Config struct { + // ServerConfig is used to set up the Firehose delivery + // endpoint. The Firehose delivery stream expects an HTTPS + // endpoint, so TLSSettings must be used to enable that. + confighttp.ServerConfig `mapstructure:",squash"` + // RecordType is the key used to determine which unmarshaler to use + // when receiving the requests. + RecordType string `mapstructure:"record_type"` + // AccessKey is checked against the one received with each request. + // This can be set when creating or updating the Firehose delivery + // stream. + AccessKey configopaque.String `mapstructure:"access_key"` +} + +// Validate checks that the endpoint and record type exist and +// are valid. +func (c *Config) Validate() error { + if c.Endpoint == "" { + return errors.New("must specify endpoint") + } + // If a record type is specified, it must be valid. + // An empty string is acceptable, however, because it will use a telemetry-type-specific default. + if c.RecordType != "" { + return validateRecordType(c.RecordType) + } + return nil +} diff --git a/receiver/signozawsfirehosereceiver/config_test.go b/receiver/signozawsfirehosereceiver/config_test.go new file mode 100644 index 00000000..5d9429dd --- /dev/null +++ b/receiver/signozawsfirehosereceiver/config_test.go @@ -0,0 +1,57 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/confmap/confmaptest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/metadata" +) + +func TestLoadConfig(t *testing.T) { + for _, configType := range []string{ + "cwmetrics", "cwlogs", "otlp_v1", "invalid", + } { + t.Run(configType, func(t *testing.T) { + fileName := configType + "_config.yaml" + cm, err := confmaptest.LoadConf(filepath.Join("testdata", fileName)) + require.NoError(t, err) + + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + + sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String()) + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(cfg)) + + err = component.ValidateConfig(cfg) + if configType == "invalid" { + assert.Error(t, err) + } else { + assert.NoError(t, err) + require.Equal(t, &Config{ + RecordType: configType, + AccessKey: "some_access_key", + ServerConfig: confighttp.ServerConfig{ + Endpoint: "0.0.0.0:4433", + TLSSetting: &configtls.ServerConfig{ + Config: configtls.Config{ + CertFile: "server.crt", + KeyFile: "server.key", + }, + }, + }, + }, cfg) + } + }) + } +} diff --git a/receiver/signozawsfirehosereceiver/doc.go b/receiver/signozawsfirehosereceiver/doc.go new file mode 100644 index 00000000..0fbc6826 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/doc.go @@ -0,0 +1,16 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package awsfirehosereceiver implements a receiver that can be used to +// receive requests from the AWS Kinesis Data Firehose and transform them +// into formats usable by the Opentelemetry collector. The configuration +// determines which unmarshaler to use. Each unmarshaler is responsible for +// processing a Firehose record format that can be sent through the delivery +// stream. +// +// More details can be found at: +// https://docs.aws.amazon.com/firehose/latest/dev/httpdeliveryrequestresponse.html + +//go:generate mdatagen metadata.yaml + +package awsfirehosereceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver" diff --git a/receiver/signozawsfirehosereceiver/factory.go b/receiver/signozawsfirehosereceiver/factory.go new file mode 100644 index 00000000..5058ad42 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/factory.go @@ -0,0 +1,104 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver" + +import ( + "context" + "errors" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/testutil" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/metadata" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwmetricstream" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/otlpmetricstream" +) + +const ( + defaultEndpoint = "0.0.0.0:4433" + defaultPort = 4433 +) + +var ( + errUnrecognizedRecordType = errors.New("unrecognized record type") + availableRecordTypes = map[string]bool{ + cwmetricstream.TypeStr: true, + cwlog.TypeStr: true, + otlpmetricstream.TypeStr: true, + } +) + +// NewFactory creates a receiver factory for awsfirehose. Currently, only +// available in metrics pipelines. +func NewFactory() receiver.Factory { + return receiver.NewFactory( + metadata.Type, + createDefaultConfig, + receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability), + receiver.WithLogs(createLogsReceiver, metadata.LogsStability)) +} + +// validateRecordType checks the available record types for the +// passed in one and returns an error if not found. +func validateRecordType(recordType string) error { + if _, ok := availableRecordTypes[recordType]; !ok { + return errUnrecognizedRecordType + } + return nil +} + +// defaultMetricsUnmarshalers creates a map of the available metrics +// unmarshalers. +func defaultMetricsUnmarshalers(logger *zap.Logger) map[string]unmarshaler.MetricsUnmarshaler { + cwmsu := cwmetricstream.NewUnmarshaler(logger) + otlpv1msu := otlpmetricstream.NewUnmarshaler(logger) + return map[string]unmarshaler.MetricsUnmarshaler{ + cwmsu.Type(): cwmsu, + otlpv1msu.Type(): otlpv1msu, + } +} + +// defaultLogsUnmarshalers creates a map of the available logs unmarshalers. +func defaultLogsUnmarshalers(logger *zap.Logger) map[string]unmarshaler.LogsUnmarshaler { + u := cwlog.NewUnmarshaler(logger) + return map[string]unmarshaler.LogsUnmarshaler{ + u.Type(): u, + } +} + +// createDefaultConfig creates a default config with the endpoint set +// to port 8443 and the record type set to the CloudWatch metric stream. +func createDefaultConfig() component.Config { + return &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.EndpointForPort(defaultPort), + }, + } +} + +// createMetricsReceiver implements the CreateMetrics function type. +func createMetricsReceiver( + _ context.Context, + set receiver.Settings, + cfg component.Config, + nextConsumer consumer.Metrics, +) (receiver.Metrics, error) { + return newMetricsReceiver(cfg.(*Config), set, defaultMetricsUnmarshalers(set.Logger), nextConsumer) +} + +// createMetricsReceiver implements the CreateMetricsReceiver function type. +func createLogsReceiver( + _ context.Context, + set receiver.Settings, + cfg component.Config, + nextConsumer consumer.Logs, +) (receiver.Logs, error) { + return newLogsReceiver(cfg.(*Config), set, defaultLogsUnmarshalers(set.Logger), nextConsumer) +} diff --git a/receiver/signozawsfirehosereceiver/factory_test.go b/receiver/signozawsfirehosereceiver/factory_test.go new file mode 100644 index 00000000..c4241751 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/factory_test.go @@ -0,0 +1,50 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver/receivertest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/otlpmetricstream" +) + +func TestValidConfig(t *testing.T) { + err := componenttest.CheckConfigStruct(createDefaultConfig()) + require.NoError(t, err) +} + +func TestCreateMetrics(t *testing.T) { + r, err := createMetricsReceiver( + context.Background(), + receivertest.NewNopSettings(), + createDefaultConfig(), + consumertest.NewNop(), + ) + require.NoError(t, err) + require.NotNil(t, r) +} + +func TestCreateLogsReceiver(t *testing.T) { + r, err := createLogsReceiver( + context.Background(), + receivertest.NewNopSettings(), + createDefaultConfig(), + consumertest.NewNop(), + ) + require.NoError(t, err) + require.NotNil(t, r) +} + +func TestValidateRecordType(t *testing.T) { + require.NoError(t, validateRecordType(defaultMetricsRecordType)) + require.NoError(t, validateRecordType(defaultLogsRecordType)) + require.NoError(t, validateRecordType(otlpmetricstream.TypeStr)) + require.Error(t, validateRecordType("nop")) +} diff --git a/receiver/signozawsfirehosereceiver/generated_component_test.go b/receiver/signozawsfirehosereceiver/generated_component_test.go new file mode 100644 index 00000000..64626eb5 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/generated_component_test.go @@ -0,0 +1,76 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package awsfirehosereceiver + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, "awsfirehose", NewFactory().Type().String()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + tests := []struct { + name string + createFn func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) + }{ + + { + name: "logs", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateLogs(ctx, set, cfg, consumertest.NewNop()) + }, + }, + + { + name: "metrics", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateMetrics(ctx, set, cfg, consumertest.NewNop()) + }, + }, + } + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + + for _, tt := range tests { + t.Run(tt.name+"-shutdown", func(t *testing.T) { + c, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run(tt.name+"-lifecycle", func(t *testing.T) { + firstRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + host := componenttest.NewNopHost() + require.NoError(t, err) + require.NoError(t, firstRcvr.Start(context.Background(), host)) + require.NoError(t, firstRcvr.Shutdown(context.Background())) + secondRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + require.NoError(t, secondRcvr.Start(context.Background(), host)) + require.NoError(t, secondRcvr.Shutdown(context.Background())) + }) + } +} diff --git a/receiver/signozawsfirehosereceiver/generated_package_test.go b/receiver/signozawsfirehosereceiver/generated_package_test.go new file mode 100644 index 00000000..4de8a57d --- /dev/null +++ b/receiver/signozawsfirehosereceiver/generated_package_test.go @@ -0,0 +1,13 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package awsfirehosereceiver + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/receiver/signozawsfirehosereceiver/go.mod b/receiver/signozawsfirehosereceiver/go.mod new file mode 100644 index 00000000..900f9ba2 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/go.mod @@ -0,0 +1,83 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver + +go 1.22.0 + +require ( + github.com/gogo/protobuf v1.3.2 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.114.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/component v0.114.0 + go.opentelemetry.io/collector/component/componentstatus v0.114.0 + go.opentelemetry.io/collector/component/componenttest v0.114.0 + go.opentelemetry.io/collector/config/confighttp v0.114.0 + go.opentelemetry.io/collector/config/configopaque v1.20.0 + go.opentelemetry.io/collector/config/configtls v1.20.0 + go.opentelemetry.io/collector/confmap v1.20.0 + go.opentelemetry.io/collector/consumer v0.114.0 + go.opentelemetry.io/collector/consumer/consumertest v0.114.0 + go.opentelemetry.io/collector/pdata v1.20.0 + go.opentelemetry.io/collector/receiver v0.114.0 + go.opentelemetry.io/collector/receiver/receivertest v0.114.0 + go.opentelemetry.io/collector/semconv v0.114.0 + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.2 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/cors v1.11.1 // indirect + go.opentelemetry.io/collector/client v1.20.0 // indirect + go.opentelemetry.io/collector/config/configauth v0.114.0 // indirect + go.opentelemetry.io/collector/config/configcompression v1.20.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.114.0 // indirect + go.opentelemetry.io/collector/config/internal v0.114.0 // indirect + go.opentelemetry.io/collector/consumer/consumererror v0.114.0 // indirect + go.opentelemetry.io/collector/consumer/consumerprofiles v0.114.0 // indirect + go.opentelemetry.io/collector/extension v0.114.0 // indirect + go.opentelemetry.io/collector/extension/auth v0.114.0 // indirect + go.opentelemetry.io/collector/featuregate v1.20.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.114.0 // indirect + go.opentelemetry.io/collector/pipeline v0.114.0 // indirect + go.opentelemetry.io/collector/receiver/receiverprofiles v0.114.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect + go.opentelemetry.io/otel v1.32.0 // indirect + go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/sdk v1.32.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/text v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +retract ( + v0.76.2 + v0.76.1 + v0.65.0 +) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/common => ../../internal/common diff --git a/receiver/signozawsfirehosereceiver/go.sum b/receiver/signozawsfirehosereceiver/go.sum new file mode 100644 index 00000000..1fc55f1c --- /dev/null +++ b/receiver/signozawsfirehosereceiver/go.sum @@ -0,0 +1,178 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ= +github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/client v1.20.0 h1:o60wPcj5nLtaRenF+1E5p4QXFS3TDL6vHlw+GOon3rg= +go.opentelemetry.io/collector/client v1.20.0/go.mod h1:6aqkszco9FaLWCxyJEVam6PP7cUa8mPRIXeS5eZGj0U= +go.opentelemetry.io/collector/component v0.114.0 h1:SVGbm5LvHGSTEDv7p92oPuBgK5tuiWR82I9+LL4TtBE= +go.opentelemetry.io/collector/component v0.114.0/go.mod h1:MLxtjZ6UVHjDxSdhGLuJfHBHvfl1iT/Y7IaQPD24Eww= +go.opentelemetry.io/collector/component/componentstatus v0.114.0 h1:y9my/xink8KB5lK8zFAjgB2+pEh0QYy5TM972fxZY9w= +go.opentelemetry.io/collector/component/componentstatus v0.114.0/go.mod h1:RIoeCYZpPaae7QLE/1RacqzhHuXBmzRAk9H/EwYtIIs= +go.opentelemetry.io/collector/component/componenttest v0.114.0 h1:GM4FTTlfeXoVm6sZYBHImwlRN8ayh2oAfUhvaFj7Zo8= +go.opentelemetry.io/collector/component/componenttest v0.114.0/go.mod h1:ZZEJMtbJtoVC/3/9R1HzERq+cYQRxuMFQrPCpfZ4Xos= +go.opentelemetry.io/collector/config/configauth v0.114.0 h1:R2sJ6xpsLYGH0yU0vCxotzBYDKR/Hrjv0A7y9lwMyiw= +go.opentelemetry.io/collector/config/configauth v0.114.0/go.mod h1:3Z24KcCpG+WYCeQYfs/cNp5cP2BDeOqLCtOEgs/rPqM= +go.opentelemetry.io/collector/config/configcompression v1.20.0 h1:H/mvz7J/5z+O74YsO0t2tk+REnO2tzLM8TgIQ4AZ5w0= +go.opentelemetry.io/collector/config/configcompression v1.20.0/go.mod h1:pnxkFCLUZLKWzYJvfSwZnPrnm0twX14CYj2ADth5xiU= +go.opentelemetry.io/collector/config/confighttp v0.114.0 h1:DjGsBvVm+mGK3IpJBaXianWhwcxEC1fF33cpuC1LY/I= +go.opentelemetry.io/collector/config/confighttp v0.114.0/go.mod h1:nrlNLxOZ+4JQaV9j0TiqQV7LOHhrRivPrT8nQRHED3Q= +go.opentelemetry.io/collector/config/configopaque v1.20.0 h1:2I48zKiyyyYqjm7y0B9OLp24ku2ZSX3nCHG0r5FdWOQ= +go.opentelemetry.io/collector/config/configopaque v1.20.0/go.mod h1:6zlLIyOoRpJJ+0bEKrlZOZon3rOp5Jrz9fMdR4twOS4= +go.opentelemetry.io/collector/config/configtelemetry v0.114.0 h1:kjLeyrumge6wsX6ZIkicdNOlBXaEyW2PI2ZdVXz/rzY= +go.opentelemetry.io/collector/config/configtelemetry v0.114.0/go.mod h1:R0MBUxjSMVMIhljuDHWIygzzJWQyZHXXWIgQNxcFwhc= +go.opentelemetry.io/collector/config/configtls v1.20.0 h1:hNlJdwfyY5Qe54RLJ41lfLqKTn9ypkR7sk7JNCcSe2U= +go.opentelemetry.io/collector/config/configtls v1.20.0/go.mod h1:sav/txSHguadTYlSSK+BJO2ljJeYEtRoBahgzWAguYg= +go.opentelemetry.io/collector/config/internal v0.114.0 h1:uWSDWTJb8T6xRjKD9/XmEARakXnxgYVYKUeId78hErc= +go.opentelemetry.io/collector/config/internal v0.114.0/go.mod h1:yC7E4h1Uj0SubxcFImh6OvBHFTjMh99+A5PuyIgDWqc= +go.opentelemetry.io/collector/confmap v1.20.0 h1:ARfOwmkKxFOud1njl03yAHQ30+uenlzqCO6LBYamDTE= +go.opentelemetry.io/collector/confmap v1.20.0/go.mod h1:DMpd9Ay/ffls3JoQBQ73vWeRsz1rNuLbwjo6WtjSQus= +go.opentelemetry.io/collector/consumer v0.114.0 h1:1zVaHvfIZowGwZRitRBRo3i+RP2StlU+GClYiofSw0Q= +go.opentelemetry.io/collector/consumer v0.114.0/go.mod h1:d+Mrzt9hsH1ub3zmwSlnQVPLeTYir4Mgo7CrWfnncN4= +go.opentelemetry.io/collector/consumer/consumererror v0.114.0 h1:r2YiELfWerb40FHD23V04gNjIkLUcjEKGxI4Vtm2iO4= +go.opentelemetry.io/collector/consumer/consumererror v0.114.0/go.mod h1:MzIrLQ5jptO2egypolhlAbZsWZr29WC4FhSxQjnxcvg= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.114.0 h1:5pXYy3E6UK5Huu3aQbsYL8B6E6MyWx4fvXXDn+oXZaA= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.114.0/go.mod h1:PMq3f54KcJQO4v1tue0QxQScu7REFVADlXxXSAYMiN0= +go.opentelemetry.io/collector/consumer/consumertest v0.114.0 h1:isaTwJK5DOy8Bs7GuLq23ejfgj8gLIo5dOUvkRnLF4g= +go.opentelemetry.io/collector/consumer/consumertest v0.114.0/go.mod h1:GNeLPkfRPdh06n/Rv1UKa/cAtCKjN0a7ADyHjIj4HFE= +go.opentelemetry.io/collector/extension v0.114.0 h1:9Qb92y8hD2WDC5aMDoj4JNQN+/5BQYJWPUPzLXX+iGw= +go.opentelemetry.io/collector/extension v0.114.0/go.mod h1:Yk2/1ptVgfTr12t+22v93nYJpioP14pURv2YercSzU0= +go.opentelemetry.io/collector/extension/auth v0.114.0 h1:1K2qh4yvG8kKR/sTAobI/rw5VxzPZoKcl3FmC195vvo= +go.opentelemetry.io/collector/extension/auth v0.114.0/go.mod h1:IjtsG+jUVJB0utKF8dAK8pLutRun3aEgASshImzsw/U= +go.opentelemetry.io/collector/featuregate v1.20.0 h1:Mi7nMy/q52eruI+6jWnMKUOeM55XvwoPnGcdB1++O8c= +go.opentelemetry.io/collector/featuregate v1.20.0/go.mod h1:47xrISO71vJ83LSMm8+yIDsUbKktUp48Ovt7RR6VbRs= +go.opentelemetry.io/collector/pdata v1.20.0 h1:ePcwt4bdtISP0loHaE+C9xYoU2ZkIvWv89Fob16o9SM= +go.opentelemetry.io/collector/pdata v1.20.0/go.mod h1:Ox1YVLe87cZDB/TL30i4SUz1cA5s6AM6SpFMfY61ICs= +go.opentelemetry.io/collector/pdata/pprofile v0.114.0 h1:pUNfTzsI/JUTiE+DScDM4lsrPoxnVNLI2fbTxR/oapo= +go.opentelemetry.io/collector/pdata/pprofile v0.114.0/go.mod h1:4aNcj6WM1n1uXyFSXlhVs4ibrERgNYsTbzcYI2zGhxA= +go.opentelemetry.io/collector/pdata/testdata v0.114.0 h1:+AzszWSL1i4K6meQ8rU0JDDW55SYCXa6FVqfDixhhTo= +go.opentelemetry.io/collector/pdata/testdata v0.114.0/go.mod h1:bv8XFdCTZxG2MQB5l9dKxSxf5zBrcodwO6JOy1+AxXM= +go.opentelemetry.io/collector/pipeline v0.114.0 h1:v3YOhc5z0tD6QbO5n/pnftpIeroihM2ks9Z2yKPCcwY= +go.opentelemetry.io/collector/pipeline v0.114.0/go.mod h1:4vOvjVsoYTHVGTbfFwqfnQOSV2K3RKUHofh3jNRc2Mg= +go.opentelemetry.io/collector/receiver v0.114.0 h1:90SAnXAjNq7/k52/pFmmb06Cf1YauoPYtbio4aOXafY= +go.opentelemetry.io/collector/receiver v0.114.0/go.mod h1:KUGT0/D953LXbGH/D3lLPU8yrU3HfWnUqpt4W4hSOnE= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.114.0 h1:ibhEfGpvNB3yrtpl2jYFabrunMk1hurxvMYpM0b1Ck4= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.114.0/go.mod h1:UZyRfaasw+NLvN10AN8IQnmj5tQ3BOUH1uP2ctpO9f0= +go.opentelemetry.io/collector/receiver/receivertest v0.114.0 h1:D+Kh9t2n4asTnM+TiSxbrKlUemLZandWntj17BJWWb0= +go.opentelemetry.io/collector/receiver/receivertest v0.114.0/go.mod h1:mNSHQ13vFmqD+VAcRzLjStFBejbcWUn2Mp0pAd7Op+U= +go.opentelemetry.io/collector/semconv v0.114.0 h1:/eKcCJwZepQUtEuFuxa0thx2XIOvhFpaf214ZG1a11k= +go.opentelemetry.io/collector/semconv v0.114.0/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/receiver/signozawsfirehosereceiver/internal/metadata/generated_status.go b/receiver/signozawsfirehosereceiver/internal/metadata/generated_status.go new file mode 100644 index 00000000..447dc318 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/metadata/generated_status.go @@ -0,0 +1,17 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("awsfirehose") + ScopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver" +) + +const ( + MetricsStability = component.StabilityLevelAlpha + LogsStability = component.StabilityLevelAlpha +) diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/compression/compression.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/compression/compression.go new file mode 100644 index 00000000..2ebca778 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/compression/compression.go @@ -0,0 +1,48 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package compression // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog/compression" + +import ( + "bytes" + "compress/gzip" +) + +// Zip returns a gzip-compressed representation of the input bytes. +func Zip(data []byte) ([]byte, error) { + var b bytes.Buffer + w := gzip.NewWriter(&b) + + _, err := w.Write(data) + if err != nil { + return nil, err + } + + if err = w.Flush(); err != nil { + return nil, err + } + + if err = w.Close(); err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +// Unzip expects gzip-compressed input bytes and returns their uncompressed form. +func Unzip(data []byte) ([]byte, error) { + b := bytes.NewBuffer(data) + + r, err := gzip.NewReader(b) + if err != nil { + return nil, err + } + + var rv bytes.Buffer + _, err = rv.ReadFrom(r) + if err != nil { + return nil, err + } + + return rv.Bytes(), nil +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/cwlog.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/cwlog.go new file mode 100644 index 00000000..1ab85509 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/cwlog.go @@ -0,0 +1,17 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwlog // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog" + +type cWLog struct { + MessageType string `json:"messageType"` + Owner string `json:"owner"` + LogGroup string `json:"logGroup"` + LogStream string `json:"logStream"` + SubscriptionFilters []string `json:"subscriptionFilters"` + LogEvents []struct { + ID string `json:"id"` + Timestamp int64 `json:"timestamp"` + Message string `json:"message"` + } `json:"logEvents"` +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/logsbuilder.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/logsbuilder.go new file mode 100644 index 00000000..eae5902b --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/logsbuilder.go @@ -0,0 +1,50 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwlog // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog" + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + conventions "go.opentelemetry.io/collector/semconv/v1.6.1" +) + +const ( + attributeAWSCloudWatchLogGroupName = "aws.cloudwatch.log_group_name" + attributeAWSCloudWatchLogStreamName = "aws.cloudwatch.log_stream_name" +) + +// resourceAttributes are the CloudWatch log attributes that define a unique resource. +type resourceAttributes struct { + owner, logGroup, logStream string +} + +// resourceLogsBuilder provides convenient access to the a Resource's LogRecordSlice. +type resourceLogsBuilder struct { + rls plog.LogRecordSlice +} + +// setAttributes applies the resourceAttributes to the provided Resource. +func (ra *resourceAttributes) setAttributes(resource pcommon.Resource) { + attrs := resource.Attributes() + attrs.PutStr(conventions.AttributeCloudAccountID, ra.owner) + attrs.PutStr(attributeAWSCloudWatchLogGroupName, ra.logGroup) + attrs.PutStr(attributeAWSCloudWatchLogStreamName, ra.logStream) +} + +// newResourceLogsBuilder to capture logs for the Resource defined by the provided attributes. +func newResourceLogsBuilder(logs plog.Logs, attrs resourceAttributes) *resourceLogsBuilder { + rls := logs.ResourceLogs().AppendEmpty() + attrs.setAttributes(rls.Resource()) + return &resourceLogsBuilder{rls.ScopeLogs().AppendEmpty().LogRecords()} +} + +// AddLog events to the LogRecordSlice. Resource attributes are captured when creating +// the resourceLogsBuilder, so we only need to consider the LogEvents themselves. +func (rlb *resourceLogsBuilder) AddLog(log cWLog) { + for _, event := range log.LogEvents { + logLine := rlb.rls.AppendEmpty() + logLine.SetTimestamp(pcommon.Timestamp(event.Timestamp)) + logLine.Body().SetStr(event.Message) + } +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/invalid_records b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/invalid_records new file mode 100644 index 00000000..d2672dbd --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/invalid_records @@ -0,0 +1,4 @@ +{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} +{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} \ No newline at end of file diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_records b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_records new file mode 100644 index 00000000..18a1888d --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_records @@ -0,0 +1,2 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} \ No newline at end of file diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_resources b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_resources new file mode 100644 index 00000000..0115614f --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/multiple_resources @@ -0,0 +1,6 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test1","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695234","timestamp":1725564035523,"message":"Hello world, here is our third log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test2","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695235","timestamp":1725574035523,"message":"Hello world, here is our fourth log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test1","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695236","timestamp":1725584035523,"message":"Hello world, here is our fifth log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test2","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695237","timestamp":1725594035523,"message":"Hello world, here is our sixth log message!"}]} \ No newline at end of file diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/single_record b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/single_record new file mode 100644 index 00000000..b35e166e --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/single_record @@ -0,0 +1 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/some_invalid_records b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/some_invalid_records new file mode 100644 index 00000000..4026ad87 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/testdata/some_invalid_records @@ -0,0 +1,3 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} \ No newline at end of file diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler.go new file mode 100644 index 00000000..b3fa1321 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler.go @@ -0,0 +1,104 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwlog // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog" + +import ( + "bytes" + "encoding/json" + "errors" + + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog/compression" +) + +const ( + TypeStr = "cwlogs" + recordDelimiter = "\n" +) + +var errInvalidRecords = errors.New("record format invalid") + +// Unmarshaler for the CloudWatch Log JSON record format. +type Unmarshaler struct { + logger *zap.Logger +} + +var _ unmarshaler.LogsUnmarshaler = (*Unmarshaler)(nil) + +// NewUnmarshaler creates a new instance of the Unmarshaler. +func NewUnmarshaler(logger *zap.Logger) *Unmarshaler { + return &Unmarshaler{logger} +} + +// Unmarshal deserializes the records into cWLogs and uses the +// resourceLogsBuilder to group them into a single plog.Logs. +// Skips invalid cWLogs received in the record and +func (u Unmarshaler) Unmarshal(records [][]byte) (plog.Logs, error) { + md := plog.NewLogs() + builders := make(map[resourceAttributes]*resourceLogsBuilder) + for recordIndex, compressedRecord := range records { + record, err := compression.Unzip(compressedRecord) + if err != nil { + u.logger.Error("Failed to unzip record", + zap.Error(err), + zap.Int("record_index", recordIndex), + ) + continue + } + // Multiple logs in each record separated by newline character + for datumIndex, datum := range bytes.Split(record, []byte(recordDelimiter)) { + if len(datum) > 0 { + var log cWLog + err := json.Unmarshal(datum, &log) + if err != nil { + u.logger.Error( + "Unable to unmarshal input", + zap.Error(err), + zap.Int("datum_index", datumIndex), + zap.Int("record_index", recordIndex), + ) + continue + } + if !u.isValid(log) { + u.logger.Error( + "Invalid log", + zap.Int("datum_index", datumIndex), + zap.Int("record_index", recordIndex), + ) + continue + } + attrs := resourceAttributes{ + owner: log.Owner, + logGroup: log.LogGroup, + logStream: log.LogStream, + } + lb, ok := builders[attrs] + if !ok { + lb = newResourceLogsBuilder(md, attrs) + builders[attrs] = lb + } + lb.AddLog(log) + } + } + } + + if len(builders) == 0 { + return plog.NewLogs(), errInvalidRecords + } + + return md, nil +} + +// isValid validates that the cWLog has been unmarshalled correctly. +func (u Unmarshaler) isValid(log cWLog) bool { + return log.Owner != "" && log.LogGroup != "" && log.LogStream != "" +} + +// Type of the serialized messages. +func (u Unmarshaler) Type() string { + return TypeStr +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler_test.go new file mode 100644 index 00000000..fbd68393 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler_test.go @@ -0,0 +1,83 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwlog + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog/compression" +) + +func TestType(t *testing.T) { + unmarshaler := NewUnmarshaler(zap.NewNop()) + require.Equal(t, TypeStr, unmarshaler.Type()) +} + +func TestUnmarshal(t *testing.T) { + unmarshaler := NewUnmarshaler(zap.NewNop()) + testCases := map[string]struct { + filename string + wantResourceCount int + wantLogCount int + wantErr error + }{ + "WithMultipleRecords": { + filename: "multiple_records", + wantResourceCount: 1, + wantLogCount: 2, + }, + "WithSingleRecord": { + filename: "single_record", + wantResourceCount: 1, + wantLogCount: 1, + }, + "WithInvalidRecords": { + filename: "invalid_records", + wantErr: errInvalidRecords, + }, + "WithSomeInvalidRecords": { + filename: "some_invalid_records", + wantResourceCount: 1, + wantLogCount: 2, + }, + "WithMultipleResources": { + filename: "multiple_resources", + wantResourceCount: 3, + wantLogCount: 6, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + record, err := os.ReadFile(filepath.Join(".", "testdata", testCase.filename)) + require.NoError(t, err) + + compressedRecord, err := compression.Zip(record) + require.NoError(t, err) + records := [][]byte{compressedRecord} + + got, err := unmarshaler.Unmarshal(records) + if testCase.wantErr != nil { + require.Error(t, err) + require.Equal(t, testCase.wantErr, err) + } else { + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, testCase.wantResourceCount, got.ResourceLogs().Len()) + gotLogCount := 0 + for i := 0; i < got.ResourceLogs().Len(); i++ { + rm := got.ResourceLogs().At(i) + require.Equal(t, 1, rm.ScopeLogs().Len()) + ilm := rm.ScopeLogs().At(0) + gotLogCount += ilm.LogRecords().Len() + } + require.Equal(t, testCase.wantLogCount, gotLogCount) + } + }) + } +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/cwmetric.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/cwmetric.go new file mode 100644 index 00000000..1896fd0c --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/cwmetric.go @@ -0,0 +1,47 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwmetricstream // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwmetricstream" + +// The cWMetric is the format for the CloudWatch metric stream records. +// +// More details can be found at: +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats-json.html +type cWMetric struct { + // MetricStreamName is the name of the CloudWatch metric stream. + MetricStreamName string `json:"metric_stream_name"` + // AccountID is the AWS account ID associated with the metric. + AccountID string `json:"account_id"` + // Region is the AWS region for the metric. + Region string `json:"region"` + // Namespace is the CloudWatch namespace the metric is in. + Namespace string `json:"namespace"` + // MetricName is the name of the metric. + MetricName string `json:"metric_name"` + // Dimensions is a map of name/value pairs that help to + // differentiate a metric. + Dimensions map[string]string `json:"dimensions"` + // Timestamp is the milliseconds since epoch for + // the metric. + Timestamp int64 `json:"timestamp"` + // Value is the cWMetricValue, which has the min, max, + // sum, and count. + Value *cWMetricValue `json:"value"` + // Unit is the unit for the metric. + // + // More details can be found at: + // https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html + Unit string `json:"unit"` +} + +// The cWMetricValue is the actual values of the CloudWatch metric. +type cWMetricValue struct { + // Max is the highest value observed. + Max float64 `json:"max"` + // Min is the lowest value observed. + Min float64 `json:"min"` + // Sum is the sum of data points collected. + Sum float64 `json:"sum"` + // Count is the number of data points. + Count float64 `json:"count"` +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder.go new file mode 100644 index 00000000..af47ba4b --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder.go @@ -0,0 +1,160 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwmetricstream // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwmetricstream" + +import ( + "fmt" + "strings" + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + conventions "go.opentelemetry.io/collector/semconv/v1.27.0" +) + +const ( + attributeAWSCloudWatchMetricStreamName = "aws.cloudwatch.metric_stream_name" + dimensionInstanceID = "InstanceId" + namespaceDelimiter = "/" +) + +// resourceAttributes are the CloudWatch metric stream attributes that define a +// unique resource. +type resourceAttributes struct { + // metricStreamName is the metric stream name. + metricStreamName string + // accountID is the AWS account ID. + accountID string + // region is the AWS region. + region string + // namespace is the CloudWatch metric namespace. + namespace string +} + +// The resourceMetricsBuilder is used to aggregate metrics for the +// same resourceAttributes. +type resourceMetricsBuilder struct { + rms pmetric.MetricSlice + // metricBuilders is the map of metrics within the same + // resource group. + metricBuilders map[string]*metricBuilder +} + +// newResourceMetricsBuilder creates a resourceMetricsBuilder with the +// resourceAttributes. +func newResourceMetricsBuilder(md pmetric.Metrics, attrs resourceAttributes) *resourceMetricsBuilder { + rms := md.ResourceMetrics().AppendEmpty() + attrs.setAttributes(rms.Resource()) + return &resourceMetricsBuilder{ + rms: rms.ScopeMetrics().AppendEmpty().Metrics(), + metricBuilders: make(map[string]*metricBuilder), + } +} + +// AddMetric adds a metric to one of the metric builders based on +// the key generated for each. +func (rmb *resourceMetricsBuilder) AddMetric(metric cWMetric) { + mb, ok := rmb.metricBuilders[metric.MetricName] + if !ok { + mb = newMetricBuilder(rmb.rms, metric.MetricName, metric.Unit) + rmb.metricBuilders[metric.MetricName] = mb + } + mb.AddDataPoint(metric) +} + +// setAttributes creates a pcommon.Resource from the fields in the resourceMetricsBuilder. +func (rmb *resourceAttributes) setAttributes(resource pcommon.Resource) { + attributes := resource.Attributes() + attributes.PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAWS) + attributes.PutStr(conventions.AttributeCloudAccountID, rmb.accountID) + attributes.PutStr(conventions.AttributeCloudRegion, rmb.region) + serviceNamespace, serviceName := toServiceAttributes(rmb.namespace) + if serviceNamespace != "" { + attributes.PutStr(conventions.AttributeServiceNamespace, serviceNamespace) + } + attributes.PutStr(conventions.AttributeServiceName, serviceName) + attributes.PutStr(attributeAWSCloudWatchMetricStreamName, rmb.metricStreamName) +} + +// toServiceAttributes splits the CloudWatch namespace into service namespace/name +// if prepended by AWS/. Otherwise, it returns the CloudWatch namespace as the +// service name with an empty service namespace +func toServiceAttributes(namespace string) (serviceNamespace, serviceName string) { + index := strings.Index(namespace, namespaceDelimiter) + if index != -1 && strings.EqualFold(namespace[:index], conventions.AttributeCloudProviderAWS) { + return namespace[:index], namespace[index+1:] + } + return "", namespace +} + +// dataPointKey combines the dimensions and timestamps to create a key +// used to prevent duplicate metrics. +type dataPointKey struct { + // timestamp is the milliseconds since epoch + timestamp int64 + // dimensions is the string representation of the metric dimensions. + // fmt guarantees key-sorted order when printing a map. + dimensions string +} + +// The metricBuilder aggregates metrics of the same name and unit +// into data points. +type metricBuilder struct { + metric pmetric.Metric + // seen is the set of added data point keys. + seen map[dataPointKey]bool +} + +// newMetricBuilder creates a metricBuilder with the name and unit. +func newMetricBuilder(rms pmetric.MetricSlice, name, unit string) *metricBuilder { + m := rms.AppendEmpty() + m.SetName(name) + m.SetUnit(unit) + m.SetEmptySummary() + return &metricBuilder{ + metric: m, + seen: make(map[dataPointKey]bool), + } +} + +// AddDataPoint adds the metric as a datapoint if a metric for that timestamp +// hasn't already been added. +func (mb *metricBuilder) AddDataPoint(metric cWMetric) { + key := dataPointKey{ + timestamp: metric.Timestamp, + dimensions: fmt.Sprint(metric.Dimensions), + } + if _, ok := mb.seen[key]; !ok { + mb.toDataPoint(mb.metric.Summary().DataPoints().AppendEmpty(), metric) + mb.seen[key] = true + } +} + +// toDataPoint converts a cWMetric into a pdata datapoint and attaches the +// dimensions as attributes. +func (mb *metricBuilder) toDataPoint(dp pmetric.SummaryDataPoint, metric cWMetric) { + dp.SetCount(uint64(metric.Value.Count)) + dp.SetSum(metric.Value.Sum) + qv := dp.QuantileValues() + min := qv.AppendEmpty() + min.SetQuantile(0) + min.SetValue(metric.Value.Min) + max := qv.AppendEmpty() + max.SetQuantile(1) + max.SetValue(metric.Value.Max) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(metric.Timestamp))) + for k, v := range metric.Dimensions { + dp.Attributes().PutStr(ToSemConvAttributeKey(k), v) + } +} + +// ToSemConvAttributeKey maps some common keys to semantic convention attributes. +func ToSemConvAttributeKey(key string) string { + switch key { + case dimensionInstanceID: + return conventions.AttributeServiceInstanceID + default: + return key + } +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder_test.go new file mode 100644 index 00000000..11455464 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/metricsbuilder_test.go @@ -0,0 +1,212 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwmetricstream + +import ( + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pmetric" + conventions "go.opentelemetry.io/collector/semconv/v1.27.0" +) + +const ( + testRegion = "us-east-1" + testAccountID = "1234567890" + testStreamName = "MyMetricStream" + testInstanceID = "i-1234567890abcdef0" +) + +func TestToSemConvAttributeKey(t *testing.T) { + testCases := map[string]struct { + key string + want string + }{ + "WithValidKey": { + key: "InstanceId", + want: conventions.AttributeServiceInstanceID, + }, + "WithInvalidKey": { + key: "CustomDimension", + want: "CustomDimension", + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + got := ToSemConvAttributeKey(testCase.key) + require.Equal(t, testCase.want, got) + }) + } +} + +func TestMetricBuilder(t *testing.T) { + t.Run("WithSingleMetric", func(t *testing.T) { + metric := cWMetric{ + MetricName: "name", + Unit: "unit", + Timestamp: time.Now().UnixMilli(), + Value: testCWMetricValue(), + Dimensions: map[string]string{"CustomDimension": "test"}, + } + gots := pmetric.NewMetricSlice() + mb := newMetricBuilder(gots, metric.MetricName, metric.Unit) + mb.AddDataPoint(metric) + require.Equal(t, 1, gots.Len()) + got := gots.At(0) + require.Equal(t, metric.MetricName, got.Name()) + require.Equal(t, metric.Unit, got.Unit()) + require.Equal(t, pmetric.MetricTypeSummary, got.Type()) + gotDps := got.Summary().DataPoints() + require.Equal(t, 1, gotDps.Len()) + gotDp := gotDps.At(0) + require.Equal(t, uint64(metric.Value.Count), gotDp.Count()) + require.Equal(t, metric.Value.Sum, gotDp.Sum()) + gotQv := gotDp.QuantileValues() + require.Equal(t, 2, gotQv.Len()) + require.Equal(t, []float64{metric.Value.Min, metric.Value.Max}, []float64{gotQv.At(0).Value(), gotQv.At(1).Value()}) + require.Equal(t, 1, gotDp.Attributes().Len()) + }) + t.Run("WithTimestampCollision", func(t *testing.T) { + timestamp := time.Now().UnixMilli() + metrics := []cWMetric{ + { + Timestamp: timestamp, + Value: testCWMetricValue(), + Dimensions: map[string]string{ + "AccountId": testAccountID, + "Region": testRegion, + "InstanceId": testInstanceID, + }, + }, + { + Timestamp: timestamp, + Value: testCWMetricValue(), + Dimensions: map[string]string{ + "InstanceId": testInstanceID, + "AccountId": testAccountID, + "Region": testRegion, + }, + }, + } + gots := pmetric.NewMetricSlice() + mb := newMetricBuilder(gots, "name", "unit") + for _, metric := range metrics { + mb.AddDataPoint(metric) + } + require.Equal(t, 1, gots.Len()) + got := gots.At(0) + gotDps := got.Summary().DataPoints() + require.Equal(t, 1, gotDps.Len()) + gotDp := gotDps.At(0) + require.Equal(t, uint64(metrics[0].Value.Count), gotDp.Count()) + require.Equal(t, metrics[0].Value.Sum, gotDp.Sum()) + require.Equal(t, 3, gotDp.Attributes().Len()) + }) +} + +func TestResourceMetricsBuilder(t *testing.T) { + testCases := map[string]struct { + namespace string + wantAttributes map[string]string + }{ + "WithAwsNamespace": { + namespace: "AWS/EC2", + wantAttributes: map[string]string{ + attributeAWSCloudWatchMetricStreamName: testStreamName, + conventions.AttributeCloudAccountID: testAccountID, + conventions.AttributeCloudRegion: testRegion, + conventions.AttributeServiceName: "EC2", + conventions.AttributeServiceNamespace: "AWS", + }, + }, + "WithCustomNamespace": { + namespace: "CustomNamespace", + wantAttributes: map[string]string{ + attributeAWSCloudWatchMetricStreamName: testStreamName, + conventions.AttributeCloudAccountID: testAccountID, + conventions.AttributeCloudRegion: testRegion, + conventions.AttributeServiceName: "CustomNamespace", + conventions.AttributeServiceNamespace: "", + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + metric := cWMetric{ + MetricName: "name", + Unit: "unit", + Timestamp: time.Now().UnixMilli(), + Value: testCWMetricValue(), + Dimensions: map[string]string{}, + } + attrs := resourceAttributes{ + metricStreamName: testStreamName, + accountID: testAccountID, + region: testRegion, + namespace: testCase.namespace, + } + gots := pmetric.NewMetrics() + rmb := newResourceMetricsBuilder(gots, attrs) + rmb.AddMetric(metric) + require.Equal(t, 1, gots.ResourceMetrics().Len()) + got := gots.ResourceMetrics().At(0) + gotAttrs := got.Resource().Attributes() + for wantKey, wantValue := range testCase.wantAttributes { + gotValue, ok := gotAttrs.Get(wantKey) + if wantValue != "" { + require.True(t, ok) + require.Equal(t, wantValue, gotValue.AsString()) + } else { + require.False(t, ok) + } + } + }) + } + t.Run("WithSameMetricDifferentDimensions", func(t *testing.T) { + metrics := []cWMetric{ + { + MetricName: "name", + Unit: "unit", + Timestamp: time.Now().UnixMilli(), + Value: testCWMetricValue(), + Dimensions: map[string]string{}, + }, + { + MetricName: "name", + Unit: "unit", + Timestamp: time.Now().Add(time.Second * 3).UnixMilli(), + Value: testCWMetricValue(), + Dimensions: map[string]string{ + "CustomDimension": "value", + }, + }, + } + attrs := resourceAttributes{ + metricStreamName: testStreamName, + accountID: testAccountID, + region: testRegion, + namespace: "AWS/EC2", + } + gots := pmetric.NewMetrics() + rmb := newResourceMetricsBuilder(gots, attrs) + for _, metric := range metrics { + rmb.AddMetric(metric) + } + require.Equal(t, 1, gots.ResourceMetrics().Len()) + got := gots.ResourceMetrics().At(0) + require.Equal(t, 1, got.ScopeMetrics().Len()) + gotMetrics := got.ScopeMetrics().At(0).Metrics() + require.Equal(t, 1, gotMetrics.Len()) + gotDps := gotMetrics.At(0).Summary().DataPoints() + require.Equal(t, 2, gotDps.Len()) + }) +} + +// testCWMetricValue is a convenience function for creating a test cWMetricValue +func testCWMetricValue() *cWMetricValue { + return &cWMetricValue{100, 0, float64(rand.Int63n(100)), float64(rand.Int63n(4))} +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/package_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/package_test.go new file mode 100644 index 00000000..18446de0 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwmetricstream + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/invalid_records b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/invalid_records new file mode 100644 index 00000000..d2672dbd --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/invalid_records @@ -0,0 +1,4 @@ +{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} +{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} \ No newline at end of file diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/multiple_records b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/multiple_records new file mode 100644 index 00000000..20fa9b36 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/multiple_records @@ -0,0 +1,127 @@ +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionEstablishedCount","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":477184.0,"count":92.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":40.0,"min":40.0,"sum":40.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":59.98,"min":59.98,"sum":59.98,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{},"timestamp":1643916720000,"value":{"max":47.0,"min":41.0,"sum":88.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":96768.0,"min":96768.0,"sum":96768.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916720000,"value":{"max":47.0,"min":41.0,"sum":88.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUSurplusCreditsCharged","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.00133333333333333,"min":0.00133333333333333,"sum":0.00133333333333333,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.006538,"min":0.006538,"sum":0.006538,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":0.028416,"min":0.028416,"sum":0.028416,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.08,"count":92.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":223744.0,"min":223744.0,"sum":223744.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUCreditUsage","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.018548,"min":0.018548,"sum":0.018548,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ActiveConnectionCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionAttemptCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":60.145003,"min":60.145003,"sum":60.145003,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"PublishErrorRate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"BurstBalance","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":100.0,"min":100.0,"sum":100.0,"count":1.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":34.0,"min":34.0,"sum":34.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"Spot","Service":"Fargate","Type":"Resource"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeInstanceStatus","Service":"EC2","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":92.0,"min":92.0,"sum":92.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":60.07399,"min":60.07399,"sum":60.07399,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionEstablishedCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"PublishErrorRate","dimensions":{},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionAttemptCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ActiveConnectionCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"ListGroups","Service":"Resource Groups","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":7.0,"min":7.0,"sum":7.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{},"timestamp":1643916780000,"value":{"max":48.0,"min":36.0,"sum":84.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":178688.0,"min":178688.0,"sum":178688.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916780000,"value":{"max":48.0,"min":36.0,"sum":84.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.018855,"min":0.018855,"sum":0.018855,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.004314,"min":0.004314,"sum":0.004314,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":40448.0,"min":40448.0,"sum":40448.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/single_record b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/single_record new file mode 100644 index 00000000..6d2fd8e1 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/single_record @@ -0,0 +1 @@ +{"metric_stream_name": "MyMetricStream","account_id": "1234567890","region": "us-east-1","namespace": "AWS/EC2","metric_name": "DiskWriteOps","dimensions": {"InstanceId": "i-123456789012"},"timestamp": 1611929698000,"value": {"count": 3.0,"sum": 20.0,"max": 18.0,"min": 0.0},"unit": "Seconds"} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/some_invalid_records b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/some_invalid_records new file mode 100644 index 00000000..d1598692 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/testdata/some_invalid_records @@ -0,0 +1,92 @@ +{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} +{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":60.058357,"min":60.058357,"sum":60.058357,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":189.0,"min":187.0,"sum":376.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":2.225,"min":2.208333333333333,"sum":4.433333333333334,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":99.0,"min":99.0,"sum":198.0,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":2.225,"min":2.1,"sum":8.675,"count":4.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":37.0,"min":12.0,"sum":49.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"BurstBalance","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":100.0,"count":1.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":7.0,"min":7.0,"sum":7.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":246272.0,"min":49664.0,"sum":552448.0,"count":4.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":246272.0,"min":49664.0,"sum":295936.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"Spot","Service":"Fargate","Type":"Resource"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":189.0,"min":167.0,"sum":729.0,"count":4.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":33477.0,"min":28026.0,"sum":127491.0,"count":4.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":4.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":96546.0,"min":75023.0,"sum":330899.0,"count":4.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":172.0,"min":172.0,"sum":172.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":4.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":33477.0,"min":33411.0,"sum":66888.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":60.107249,"min":60.107249,"sum":60.107249,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":200.0,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":100.0,"min":99.0,"sum":398.0,"count":4.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"Standard/OnDemand","Resource":"vCPU","Service":"EC2","Type":"Resource"},"timestamp":1643916600000,"value":{"max":6.0,"min":6.0,"sum":6.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":50.0,"min":7.0,"sum":106.0,"count":4.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":210.0,"min":206.0,"sum":416.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":205.0,"min":182.0,"sum":387.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":186.0,"min":167.0,"sum":353.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":10.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":50.0,"min":7.0,"sum":57.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":892416.0,"count":172.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":12.0,"min":12.0,"sum":12.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":59.98,"min":59.98,"sum":59.98,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":75359.0,"min":75023.0,"sum":150382.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":49664.0,"min":49664.0,"sum":49664.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":5.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":5.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":96546.0,"min":72132.0,"sum":252649.0,"count":3.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":2.1416666666666666,"min":2.1,"sum":4.241666666666667,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":100.0,"min":99.0,"sum":497.0,"count":5.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":210.0,"min":182.0,"sum":991.0,"count":5.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":32577.0,"min":28026.0,"sum":60603.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":223744.0,"min":78336.0,"sum":480256.0,"count":3.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.00133333333333333,"min":0.00133333333333333,"sum":0.00133333333333333,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.004637,"min":0.004637,"sum":0.004637,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.00977,"min":0.00977,"sum":0.00977,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.08,"count":172.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":99.0,"min":99.0,"sum":297.0,"count":3.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":200.0,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":78336.0,"min":78336.0,"sum":78336.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler.go new file mode 100644 index 00000000..110ef4af --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler.go @@ -0,0 +1,99 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwmetricstream // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwmetricstream" + +import ( + "bytes" + "encoding/json" + "errors" + + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" +) + +const ( + TypeStr = "cwmetrics" + recordDelimiter = "\n" +) + +var errInvalidRecords = errors.New("record format invalid") + +// Unmarshaler for the CloudWatch Metric Stream JSON record format. +// +// More details can be found at: +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats-json.html +type Unmarshaler struct { + logger *zap.Logger +} + +var _ unmarshaler.MetricsUnmarshaler = (*Unmarshaler)(nil) + +// NewUnmarshaler creates a new instance of the Unmarshaler. +func NewUnmarshaler(logger *zap.Logger) *Unmarshaler { + return &Unmarshaler{logger} +} + +// Unmarshal deserializes the records into cWMetrics and uses the +// resourceMetricsBuilder to group them into a single pmetric.Metrics. +// Skips invalid cWMetrics received in the record and +func (u Unmarshaler) Unmarshal(records [][]byte) (pmetric.Metrics, error) { + md := pmetric.NewMetrics() + builders := make(map[resourceAttributes]*resourceMetricsBuilder) + for recordIndex, record := range records { + // Multiple metrics in each record separated by newline character + for datumIndex, datum := range bytes.Split(record, []byte(recordDelimiter)) { + if len(datum) > 0 { + var metric cWMetric + err := json.Unmarshal(datum, &metric) + if err != nil { + u.logger.Error( + "Unable to unmarshal input", + zap.Error(err), + zap.Int("datum_index", datumIndex), + zap.Int("record_index", recordIndex), + ) + continue + } + if !u.isValid(metric) { + u.logger.Error( + "Invalid metric", + zap.Int("datum_index", datumIndex), + zap.Int("record_index", recordIndex), + ) + continue + } + attrs := resourceAttributes{ + metricStreamName: metric.MetricStreamName, + namespace: metric.Namespace, + accountID: metric.AccountID, + region: metric.Region, + } + mb, ok := builders[attrs] + if !ok { + mb = newResourceMetricsBuilder(md, attrs) + builders[attrs] = mb + } + mb.AddMetric(metric) + } + } + } + + if len(builders) == 0 { + return pmetric.NewMetrics(), errInvalidRecords + } + + return md, nil +} + +// isValid validates that the cWMetric has been unmarshalled correctly. +func (u Unmarshaler) isValid(metric cWMetric) bool { + return metric.MetricName != "" && metric.Namespace != "" && metric.Unit != "" && metric.Value != nil +} + +// Type of the serialized messages. +func (u Unmarshaler) Type() string { + return TypeStr +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler_test.go new file mode 100644 index 00000000..16a517db --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/cwmetricstream/unmarshaler_test.go @@ -0,0 +1,84 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cwmetricstream + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestType(t *testing.T) { + unmarshaler := NewUnmarshaler(zap.NewNop()) + require.Equal(t, TypeStr, unmarshaler.Type()) +} + +func TestUnmarshal(t *testing.T) { + unmarshaler := NewUnmarshaler(zap.NewNop()) + testCases := map[string]struct { + filename string + wantResourceCount int + wantMetricCount int + wantDatapointCount int + wantErr error + }{ + "WithMultipleRecords": { + filename: "multiple_records", + wantResourceCount: 6, + wantMetricCount: 33, + wantDatapointCount: 127, + }, + "WithSingleRecord": { + filename: "single_record", + wantResourceCount: 1, + wantMetricCount: 1, + wantDatapointCount: 1, + }, + "WithInvalidRecords": { + filename: "invalid_records", + wantErr: errInvalidRecords, + }, + "WithSomeInvalidRecords": { + filename: "some_invalid_records", + wantResourceCount: 5, + wantMetricCount: 35, + wantDatapointCount: 88, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + record, err := os.ReadFile(filepath.Join(".", "testdata", testCase.filename)) + require.NoError(t, err) + + records := [][]byte{record} + + got, err := unmarshaler.Unmarshal(records) + if testCase.wantErr != nil { + require.Error(t, err) + require.Equal(t, testCase.wantErr, err) + } else { + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, testCase.wantResourceCount, got.ResourceMetrics().Len()) + gotMetricCount := 0 + gotDatapointCount := 0 + for i := 0; i < got.ResourceMetrics().Len(); i++ { + rm := got.ResourceMetrics().At(i) + require.Equal(t, 1, rm.ScopeMetrics().Len()) + ilm := rm.ScopeMetrics().At(0) + gotMetricCount += ilm.Metrics().Len() + for j := 0; j < ilm.Metrics().Len(); j++ { + metric := ilm.Metrics().At(j) + gotDatapointCount += metric.Summary().DataPoints().Len() + } + } + require.Equal(t, testCase.wantMetricCount, gotMetricCount) + require.Equal(t, testCase.wantDatapointCount, gotDatapointCount) + } + }) + } +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler.go new file mode 100644 index 00000000..c3dde969 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler.go @@ -0,0 +1,71 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package otlpmetricstream // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/otlpmetricstream" + +import ( + "errors" + + "github.com/gogo/protobuf/proto" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" +) + +const ( + // Supported version depends on version of go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp dependency + TypeStr = "otlp_v1" +) + +var errInvalidOTLPFormatStart = errors.New("unable to decode data length from message") + +// Unmarshaler for the CloudWatch Metric Stream OpenTelemetry record format. +// +// More details can be found at: +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats-opentelemetry-100.html +type Unmarshaler struct { + logger *zap.Logger +} + +var _ unmarshaler.MetricsUnmarshaler = (*Unmarshaler)(nil) + +// NewUnmarshaler creates a new instance of the Unmarshaler. +func NewUnmarshaler(logger *zap.Logger) *Unmarshaler { + return &Unmarshaler{logger} +} + +// Unmarshal deserializes the records into pmetric.Metrics +func (u Unmarshaler) Unmarshal(records [][]byte) (pmetric.Metrics, error) { + md := pmetric.NewMetrics() + for recordIndex, record := range records { + dataLen, pos := len(record), 0 + for pos < dataLen { + n, nLen := proto.DecodeVarint(record) + if nLen == 0 && n == 0 { + return md, errInvalidOTLPFormatStart + } + req := pmetricotlp.NewExportRequest() + pos += nLen + err := req.UnmarshalProto(record[pos : pos+int(n)]) + pos += int(n) + if err != nil { + u.logger.Error( + "Unable to unmarshal input", + zap.Error(err), + zap.Int("record_index", recordIndex), + ) + continue + } + req.Metrics().ResourceMetrics().MoveAndAppendTo(md.ResourceMetrics()) + } + } + + return md, nil +} + +// Type of the serialized messages. +func (u Unmarshaler) Type() string { + return TypeStr +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler_test.go new file mode 100644 index 00000000..0f5a1ee3 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/otlpmetricstream/unmarshaler_test.go @@ -0,0 +1,125 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package otlpmetricstream + +import ( + "testing" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" + "go.uber.org/zap" +) + +func TestType(t *testing.T) { + unmarshaler := NewUnmarshaler(zap.NewNop()) + require.Equal(t, TypeStr, unmarshaler.Type()) +} + +func createMetricRecord() []byte { + er := pmetricotlp.NewExportRequest() + rsm := er.Metrics().ResourceMetrics().AppendEmpty() + sm := rsm.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + sm.SetName("TestMetric") + dp := sm.SetEmptySummary().DataPoints().AppendEmpty() + dp.SetCount(1) + dp.SetSum(1) + qv := dp.QuantileValues() + min := qv.AppendEmpty() + min.SetQuantile(0) + min.SetValue(0) + max := qv.AppendEmpty() + max.SetQuantile(1) + max.SetValue(1) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + + temp, _ := er.MarshalProto() + record := proto.EncodeVarint(uint64(len(temp))) + record = append(record, temp...) + return record +} + +func TestUnmarshal(t *testing.T) { + unmarshaler := NewUnmarshaler(zap.NewNop()) + testCases := map[string]struct { + records [][]byte + wantResourceCount int + wantMetricCount int + wantDatapointCount int + wantErr error + }{ + "WithSingleRecord": { + records: [][]byte{ + createMetricRecord(), + }, + wantResourceCount: 1, + wantMetricCount: 1, + wantDatapointCount: 1, + }, + "WithMultipleRecords": { + records: [][]byte{ + createMetricRecord(), + createMetricRecord(), + createMetricRecord(), + createMetricRecord(), + createMetricRecord(), + createMetricRecord(), + }, + wantResourceCount: 6, + wantMetricCount: 6, + wantDatapointCount: 6, + }, + "WithEmptyRecord": { + records: make([][]byte, 0), + wantResourceCount: 0, + wantMetricCount: 0, + wantDatapointCount: 0, + }, + "WithInvalidRecords": { + records: [][]byte{{1, 2}}, + wantResourceCount: 0, + wantMetricCount: 0, + wantDatapointCount: 0, + }, + "WithSomeInvalidRecords": { + records: [][]byte{ + createMetricRecord(), + {1, 2}, + createMetricRecord(), + }, + wantResourceCount: 2, + wantMetricCount: 2, + wantDatapointCount: 2, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + got, err := unmarshaler.Unmarshal(testCase.records) + if testCase.wantErr != nil { + require.Error(t, err) + require.Equal(t, testCase.wantErr, err) + } else { + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, testCase.wantResourceCount, got.ResourceMetrics().Len()) + gotMetricCount := 0 + gotDatapointCount := 0 + for i := 0; i < got.ResourceMetrics().Len(); i++ { + rm := got.ResourceMetrics().At(i) + require.Equal(t, 1, rm.ScopeMetrics().Len()) + ilm := rm.ScopeMetrics().At(0) + gotMetricCount += ilm.Metrics().Len() + for j := 0; j < ilm.Metrics().Len(); j++ { + metric := ilm.Metrics().At(j) + gotDatapointCount += metric.Summary().DataPoints().Len() + } + } + require.Equal(t, testCase.wantMetricCount, gotMetricCount) + require.Equal(t, testCase.wantDatapointCount, gotDatapointCount) + } + }) + } +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshaler.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshaler.go new file mode 100644 index 00000000..0ffb4b0a --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshaler.go @@ -0,0 +1,27 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package unmarshaler // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" + +import ( + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +// MetricsUnmarshaler deserializes the message body +type MetricsUnmarshaler interface { + // Unmarshal deserializes the records into metrics. + Unmarshal(records [][]byte) (pmetric.Metrics, error) + + // Type of the serialized messages. + Type() string +} + +// LogsUnmarshaler deserializes the message body +type LogsUnmarshaler interface { + // Unmarshal deserializes the records into logs. + Unmarshal(records [][]byte) (plog.Logs, error) + + // Type of the serialized messages. + Type() string +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler.go new file mode 100644 index 00000000..79f29cae --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler.go @@ -0,0 +1,47 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package unmarshalertest // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/unmarshalertest" + +import ( + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" +) + +// NopLogsUnmarshaler is a LogsUnmarshaler that doesn't do anything +// with the inputs and just returns the logs and error passed in. +type NopLogsUnmarshaler struct { + logs plog.Logs + err error +} + +var _ unmarshaler.LogsUnmarshaler = (*NopLogsUnmarshaler)(nil) + +// NewNopLogs provides a nop logs unmarshaler with the default +// plog.Logs and no error. +func NewNopLogs() *NopLogsUnmarshaler { + return &NopLogsUnmarshaler{} +} + +// NewWithLogs provides a nop logs unmarshaler with the passed +// in logs as the result of the Unmarshal and no error. +func NewWithLogs(logs plog.Logs) *NopLogsUnmarshaler { + return &NopLogsUnmarshaler{logs: logs} +} + +// NewErrLogs provides a nop logs unmarshaler with the passed +// in error as the Unmarshal error. +func NewErrLogs(err error) *NopLogsUnmarshaler { + return &NopLogsUnmarshaler{err: err} +} + +// Unmarshal deserializes the records into logs. +func (u *NopLogsUnmarshaler) Unmarshal([][]byte) (plog.Logs, error) { + return u.logs, u.err +} + +// Type of the serialized messages. +func (u *NopLogsUnmarshaler) Type() string { + return typeStr +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler_test.go new file mode 100644 index 00000000..ce90c351 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_logs_unmarshaler_test.go @@ -0,0 +1,41 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package unmarshalertest + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/plog" +) + +func TestNewNopLogs(t *testing.T) { + unmarshaler := NewNopLogs() + got, err := unmarshaler.Unmarshal(nil) + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, typeStr, unmarshaler.Type()) +} + +func TestNewWithLogs(t *testing.T) { + logs := plog.NewLogs() + logs.ResourceLogs().AppendEmpty() + unmarshaler := NewWithLogs(logs) + got, err := unmarshaler.Unmarshal(nil) + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, logs, got) + require.Equal(t, typeStr, unmarshaler.Type()) +} + +func TestNewErrLogs(t *testing.T) { + wantErr := errors.New("test error") + unmarshaler := NewErrLogs(wantErr) + got, err := unmarshaler.Unmarshal(nil) + require.Error(t, err) + require.Equal(t, wantErr, err) + require.NotNil(t, got) + require.Equal(t, typeStr, unmarshaler.Type()) +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler.go new file mode 100644 index 00000000..a8f5c363 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler.go @@ -0,0 +1,49 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package unmarshalertest // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/unmarshalertest" + +import ( + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" +) + +const typeStr = "nop" + +// NopMetricsUnmarshaler is a MetricsUnmarshaler that doesn't do anything +// with the inputs and just returns the metrics and error passed in. +type NopMetricsUnmarshaler struct { + metrics pmetric.Metrics + err error +} + +var _ unmarshaler.MetricsUnmarshaler = (*NopMetricsUnmarshaler)(nil) + +// NewNopMetrics provides a nop metrics unmarshaler with the default +// pmetric.Metrics and no error. +func NewNopMetrics() *NopMetricsUnmarshaler { + return &NopMetricsUnmarshaler{} +} + +// NewWithMetrics provides a nop metrics unmarshaler with the passed +// in metrics as the result of the Unmarshal and no error. +func NewWithMetrics(metrics pmetric.Metrics) *NopMetricsUnmarshaler { + return &NopMetricsUnmarshaler{metrics: metrics} +} + +// NewErrMetrics provides a nop metrics unmarshaler with the passed +// in error as the Unmarshal error. +func NewErrMetrics(err error) *NopMetricsUnmarshaler { + return &NopMetricsUnmarshaler{err: err} +} + +// Unmarshal deserializes the records into metrics. +func (u *NopMetricsUnmarshaler) Unmarshal([][]byte) (pmetric.Metrics, error) { + return u.metrics, u.err +} + +// Type of the serialized messages. +func (u *NopMetricsUnmarshaler) Type() string { + return typeStr +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler_test.go new file mode 100644 index 00000000..572c39bc --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/nop_metrics_unmarshaler_test.go @@ -0,0 +1,41 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package unmarshalertest + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +func TestNewNopMetrics(t *testing.T) { + unmarshaler := NewNopMetrics() + got, err := unmarshaler.Unmarshal(nil) + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, typeStr, unmarshaler.Type()) +} + +func TestNewWithMetrics(t *testing.T) { + metrics := pmetric.NewMetrics() + metrics.ResourceMetrics().AppendEmpty() + unmarshaler := NewWithMetrics(metrics) + got, err := unmarshaler.Unmarshal(nil) + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, metrics, got) + require.Equal(t, typeStr, unmarshaler.Type()) +} + +func TestNewErrMetrics(t *testing.T) { + wantErr := errors.New("test error") + unmarshaler := NewErrMetrics(wantErr) + got, err := unmarshaler.Unmarshal(nil) + require.Error(t, err) + require.Equal(t, wantErr, err) + require.NotNil(t, got) + require.Equal(t, typeStr, unmarshaler.Type()) +} diff --git a/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/package_test.go b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/package_test.go new file mode 100644 index 00000000..21309549 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/internal/unmarshaler/unmarshalertest/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package unmarshalertest + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/receiver/signozawsfirehosereceiver/logs_receiver.go b/receiver/signozawsfirehosereceiver/logs_receiver.go new file mode 100644 index 00000000..2bf02f3b --- /dev/null +++ b/receiver/signozawsfirehosereceiver/logs_receiver.go @@ -0,0 +1,86 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver" + +import ( + "context" + "net/http" + + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog" +) + +const defaultLogsRecordType = cwlog.TypeStr + +// logsConsumer implements the firehoseConsumer +// to use a logs consumer and unmarshaler. +type logsConsumer struct { + // consumer passes the translated logs on to the + // next consumer. + consumer consumer.Logs + // unmarshaler is the configured LogsUnmarshaler + // to use when processing the records. + unmarshaler unmarshaler.LogsUnmarshaler +} + +var _ firehoseConsumer = (*logsConsumer)(nil) + +// newLogsReceiver creates a new instance of the receiver +// with a logsConsumer. +func newLogsReceiver( + config *Config, + set receiver.Settings, + unmarshalers map[string]unmarshaler.LogsUnmarshaler, + nextConsumer consumer.Logs, +) (receiver.Logs, error) { + recordType := config.RecordType + if recordType == "" { + recordType = defaultLogsRecordType + } + configuredUnmarshaler := unmarshalers[recordType] + if configuredUnmarshaler == nil { + return nil, errUnrecognizedRecordType + } + + mc := &logsConsumer{ + consumer: nextConsumer, + unmarshaler: configuredUnmarshaler, + } + + return &firehoseReceiver{ + settings: set, + config: config, + consumer: mc, + }, nil +} + +// Consume uses the configured unmarshaler to deserialize the records into a +// single plog.Logs. It will send the final result +// to the next consumer. +func (mc *logsConsumer) Consume(ctx context.Context, records [][]byte, commonAttributes map[string]string) (int, error) { + md, err := mc.unmarshaler.Unmarshal(records) + if err != nil { + return http.StatusBadRequest, err + } + + if commonAttributes != nil { + for i := 0; i < md.ResourceLogs().Len(); i++ { + rm := md.ResourceLogs().At(i) + for k, v := range commonAttributes { + if _, found := rm.Resource().Attributes().Get(k); !found { + rm.Resource().Attributes().PutStr(k, v) + } + } + } + } + + err = mc.consumer.ConsumeLogs(ctx, md) + if err != nil { + return http.StatusInternalServerError, err + } + return http.StatusOK, nil +} diff --git a/receiver/signozawsfirehosereceiver/logs_receiver_test.go b/receiver/signozawsfirehosereceiver/logs_receiver_test.go new file mode 100644 index 00000000..da448640 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/logs_receiver_test.go @@ -0,0 +1,121 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/receiver/receivertest" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/unmarshalertest" +) + +type logsRecordConsumer struct { + result plog.Logs +} + +var _ consumer.Logs = (*logsRecordConsumer)(nil) + +func (rc *logsRecordConsumer) ConsumeLogs(_ context.Context, logs plog.Logs) error { + rc.result = logs + return nil +} + +func (rc *logsRecordConsumer) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: false} +} + +func TestNewLogsReceiver(t *testing.T) { + testCases := map[string]struct { + consumer consumer.Logs + recordType string + wantErr error + }{ + "WithInvalidRecordType": { + consumer: consumertest.NewNop(), + recordType: "test", + wantErr: errUnrecognizedRecordType, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + cfg := createDefaultConfig().(*Config) + cfg.RecordType = testCase.recordType + got, err := newLogsReceiver( + cfg, + receivertest.NewNopSettings(), + defaultLogsUnmarshalers(zap.NewNop()), + testCase.consumer, + ) + require.Equal(t, testCase.wantErr, err) + if testCase.wantErr == nil { + require.NotNil(t, got) + } else { + require.Nil(t, got) + } + }) + } +} + +func TestLogsConsumer(t *testing.T) { + testErr := errors.New("test error") + testCases := map[string]struct { + unmarshalerErr error + consumerErr error + wantStatus int + wantErr error + }{ + "WithUnmarshalerError": { + unmarshalerErr: testErr, + wantStatus: http.StatusBadRequest, + wantErr: testErr, + }, + "WithConsumerError": { + consumerErr: testErr, + wantStatus: http.StatusInternalServerError, + wantErr: testErr, + }, + "WithNoError": { + wantStatus: http.StatusOK, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + mc := &logsConsumer{ + unmarshaler: unmarshalertest.NewErrLogs(testCase.unmarshalerErr), + consumer: consumertest.NewErr(testCase.consumerErr), + } + gotStatus, gotErr := mc.Consume(context.TODO(), nil, nil) + require.Equal(t, testCase.wantStatus, gotStatus) + require.Equal(t, testCase.wantErr, gotErr) + }) + } + + t.Run("WithCommonAttributes", func(t *testing.T) { + base := plog.NewLogs() + base.ResourceLogs().AppendEmpty() + rc := logsRecordConsumer{} + mc := &logsConsumer{ + unmarshaler: unmarshalertest.NewWithLogs(base), + consumer: &rc, + } + gotStatus, gotErr := mc.Consume(context.TODO(), nil, map[string]string{ + "CommonAttributes": "Test", + }) + require.Equal(t, http.StatusOK, gotStatus) + require.NoError(t, gotErr) + gotRms := rc.result.ResourceLogs() + require.Equal(t, 1, gotRms.Len()) + gotRm := gotRms.At(0) + require.Equal(t, 1, gotRm.Resource().Attributes().Len()) + }) +} diff --git a/receiver/signozawsfirehosereceiver/metadata.yaml b/receiver/signozawsfirehosereceiver/metadata.yaml new file mode 100644 index 00000000..b712ca7e --- /dev/null +++ b/receiver/signozawsfirehosereceiver/metadata.yaml @@ -0,0 +1,12 @@ +type: awsfirehose + +status: + class: receiver + stability: + alpha: [metrics, logs] + distributions: [contrib] + codeowners: + active: [Aneurysm9] + +tests: + config: diff --git a/receiver/signozawsfirehosereceiver/metrics_receiver.go b/receiver/signozawsfirehosereceiver/metrics_receiver.go new file mode 100644 index 00000000..e1eb841f --- /dev/null +++ b/receiver/signozawsfirehosereceiver/metrics_receiver.go @@ -0,0 +1,88 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver" + +import ( + "context" + "fmt" + "net/http" + + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwmetricstream" +) + +const defaultMetricsRecordType = cwmetricstream.TypeStr + +// The metricsConsumer implements the firehoseConsumer +// to use a metrics consumer and unmarshaler. +type metricsConsumer struct { + // consumer passes the translated metrics on to the + // next consumer. + consumer consumer.Metrics + // unmarshaler is the configured MetricsUnmarshaler + // to use when processing the records. + unmarshaler unmarshaler.MetricsUnmarshaler +} + +var _ firehoseConsumer = (*metricsConsumer)(nil) + +// newMetricsReceiver creates a new instance of the receiver +// with a metricsConsumer. +func newMetricsReceiver( + config *Config, + set receiver.Settings, + unmarshalers map[string]unmarshaler.MetricsUnmarshaler, + nextConsumer consumer.Metrics, +) (receiver.Metrics, error) { + recordType := config.RecordType + if recordType == "" { + recordType = defaultMetricsRecordType + } + configuredUnmarshaler := unmarshalers[recordType] + if configuredUnmarshaler == nil { + return nil, fmt.Errorf("%w: recordType = %s", errUnrecognizedRecordType, recordType) + } + + mc := &metricsConsumer{ + consumer: nextConsumer, + unmarshaler: configuredUnmarshaler, + } + + return &firehoseReceiver{ + settings: set, + config: config, + consumer: mc, + }, nil +} + +// Consume uses the configured unmarshaler to deserialize the records into a +// single pmetric.Metrics. If there are common attributes available, then it will +// attach those to each of the pcommon.Resources. It will send the final result +// to the next consumer. +func (mc *metricsConsumer) Consume(ctx context.Context, records [][]byte, commonAttributes map[string]string) (int, error) { + md, err := mc.unmarshaler.Unmarshal(records) + if err != nil { + return http.StatusBadRequest, err + } + + if commonAttributes != nil { + for i := 0; i < md.ResourceMetrics().Len(); i++ { + rm := md.ResourceMetrics().At(i) + for k, v := range commonAttributes { + if _, found := rm.Resource().Attributes().Get(k); !found { + rm.Resource().Attributes().PutStr(k, v) + } + } + } + } + + err = mc.consumer.ConsumeMetrics(ctx, md) + if err != nil { + return http.StatusInternalServerError, err + } + return http.StatusOK, nil +} diff --git a/receiver/signozawsfirehosereceiver/metrics_receiver_test.go b/receiver/signozawsfirehosereceiver/metrics_receiver_test.go new file mode 100644 index 00000000..efe6bf7c --- /dev/null +++ b/receiver/signozawsfirehosereceiver/metrics_receiver_test.go @@ -0,0 +1,122 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/receiver/receivertest" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/unmarshalertest" +) + +type recordConsumer struct { + result pmetric.Metrics +} + +var _ consumer.Metrics = (*recordConsumer)(nil) + +func (rc *recordConsumer) ConsumeMetrics(_ context.Context, metrics pmetric.Metrics) error { + rc.result = metrics + return nil +} + +func (rc *recordConsumer) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: false} +} + +func TestNewMetricsReceiver(t *testing.T) { + testCases := map[string]struct { + consumer consumer.Metrics + recordType string + wantErr error + }{ + "WithInvalidRecordType": { + consumer: consumertest.NewNop(), + recordType: "test", + wantErr: fmt.Errorf("%w: recordType = %s", errUnrecognizedRecordType, "test"), + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + cfg := createDefaultConfig().(*Config) + cfg.RecordType = testCase.recordType + got, err := newMetricsReceiver( + cfg, + receivertest.NewNopSettings(), + defaultMetricsUnmarshalers(zap.NewNop()), + testCase.consumer, + ) + require.Equal(t, testCase.wantErr, err) + if testCase.wantErr == nil { + require.NotNil(t, got) + } else { + require.Nil(t, got) + } + }) + } +} + +func TestMetricsConsumer(t *testing.T) { + testErr := errors.New("test error") + testCases := map[string]struct { + unmarshalerErr error + consumerErr error + wantStatus int + wantErr error + }{ + "WithUnmarshalerError": { + unmarshalerErr: testErr, + wantStatus: http.StatusBadRequest, + wantErr: testErr, + }, + "WithConsumerError": { + consumerErr: testErr, + wantStatus: http.StatusInternalServerError, + wantErr: testErr, + }, + "WithNoError": { + wantStatus: http.StatusOK, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + mc := &metricsConsumer{ + unmarshaler: unmarshalertest.NewErrMetrics(testCase.unmarshalerErr), + consumer: consumertest.NewErr(testCase.consumerErr), + } + gotStatus, gotErr := mc.Consume(context.TODO(), nil, nil) + require.Equal(t, testCase.wantStatus, gotStatus) + require.Equal(t, testCase.wantErr, gotErr) + }) + } + + t.Run("WithCommonAttributes", func(t *testing.T) { + base := pmetric.NewMetrics() + base.ResourceMetrics().AppendEmpty() + rc := recordConsumer{} + mc := &metricsConsumer{ + unmarshaler: unmarshalertest.NewWithMetrics(base), + consumer: &rc, + } + gotStatus, gotErr := mc.Consume(context.TODO(), nil, map[string]string{ + "CommonAttributes": "Test", + }) + require.Equal(t, http.StatusOK, gotStatus) + require.NoError(t, gotErr) + gotRms := rc.result.ResourceMetrics() + require.Equal(t, 1, gotRms.Len()) + gotRm := gotRms.At(0) + require.Equal(t, 1, gotRm.Resource().Attributes().Len()) + }) +} diff --git a/receiver/signozawsfirehosereceiver/receiver.go b/receiver/signozawsfirehosereceiver/receiver.go new file mode 100644 index 00000000..baa9750b --- /dev/null +++ b/receiver/signozawsfirehosereceiver/receiver.go @@ -0,0 +1,293 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver" + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strconv" + "sync" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componentstatus" + "go.opentelemetry.io/collector/receiver" + "go.uber.org/zap" +) + +const ( + headerFirehoseRequestID = "X-Amz-Firehose-Request-Id" + headerFirehoseAccessKey = "X-Amz-Firehose-Access-Key" + headerFirehoseCommonAttributes = "X-Amz-Firehose-Common-Attributes" + headerContentType = "Content-Type" + headerContentLength = "Content-Length" +) + +var ( + errMissingHost = errors.New("nil host") + errInvalidAccessKey = errors.New("invalid firehose access key") + errInHeaderMissingRequestID = errors.New("missing request id in header") + errInBodyMissingRequestID = errors.New("missing request id in body") + errInBodyDiffRequestID = errors.New("different request id in body") +) + +// The firehoseConsumer is responsible for using the unmarshaler and the consumer. +type firehoseConsumer interface { + // Consume unmarshalls and consumes the records. + Consume(ctx context.Context, records [][]byte, commonAttributes map[string]string) (int, error) +} + +// firehoseReceiver +type firehoseReceiver struct { + // settings is the base receiver settings. + settings receiver.Settings + // config is the configuration for the receiver. + config *Config + // server is the HTTP/HTTPS server set up to listen + // for requests. + server *http.Server + // shutdownWG is the WaitGroup that is used to wait until + // the server shutdown has completed. + shutdownWG sync.WaitGroup + // consumer is the firehoseConsumer to use to process/send + // the records in each request. + consumer firehoseConsumer +} + +// The firehoseRequest is the format of the received request body. +type firehoseRequest struct { + // RequestID is a GUID that should be the same value as + // the one in the header. + RequestID string `json:"requestId"` + // Timestamp is the milliseconds since epoch for when the + // request was generated. + Timestamp int64 `json:"timestamp"` + // Records contains the data. + Records []firehoseRecord `json:"records"` +} + +// The firehoseRecord is an individual record within the firehoseRequest. +type firehoseRecord struct { + // Data is a base64 encoded string. Can be empty. + Data string `json:"data"` +} + +// The firehoseResponse is the expected body for the response back to +// the delivery stream. +type firehoseResponse struct { + // RequestID is the same GUID that was received in + // the request. + RequestID string `json:"requestId"` + // Timestamp is the milliseconds since epoch for when the + // request finished being processed. + Timestamp int64 `json:"timestamp"` + // ErrorMessage is the error to report. Empty if request + // was successfully processed. + ErrorMessage string `json:"errorMessage,omitempty"` +} + +// The firehoseCommonAttributes is the format for the common attributes +// found in the header of requests. +type firehoseCommonAttributes struct { + // CommonAttributes can be set when creating the delivery stream. + // These will be passed to the firehoseConsumer, which should + // attach the attributes. + CommonAttributes map[string]string `json:"commonAttributes"` +} + +var ( + _ receiver.Metrics = (*firehoseReceiver)(nil) + _ http.Handler = (*firehoseReceiver)(nil) +) + +// Start spins up the receiver's HTTP server and makes the receiver start +// its processing. +func (fmr *firehoseReceiver) Start(ctx context.Context, host component.Host) error { + if host == nil { + return errMissingHost + } + + var err error + fmr.server, err = fmr.config.ServerConfig.ToServer(ctx, host, fmr.settings.TelemetrySettings, fmr) + if err != nil { + return err + } + + var listener net.Listener + listener, err = fmr.config.ServerConfig.ToListener(ctx) + if err != nil { + return err + } + fmr.shutdownWG.Add(1) + go func() { + defer fmr.shutdownWG.Done() + + if errHTTP := fmr.server.Serve(listener); errHTTP != nil && !errors.Is(errHTTP, http.ErrServerClosed) { + componentstatus.ReportStatus(host, componentstatus.NewFatalErrorEvent(errHTTP)) + } + }() + + return nil +} + +// Shutdown tells the receiver that should stop reception, +// giving it a chance to perform any necessary clean-up and +// shutting down its HTTP server. +func (fmr *firehoseReceiver) Shutdown(context.Context) error { + if fmr.server == nil { + return nil + } + err := fmr.server.Close() + fmr.shutdownWG.Wait() + return err +} + +// ServeHTTP receives Firehose requests, unmarshalls them, and sends them along to the firehoseConsumer, +// which is responsible for unmarshalling the records and sending them to the next consumer. +func (fmr *firehoseReceiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + requestID := r.Header.Get(headerFirehoseRequestID) + if requestID == "" { + fmr.settings.Logger.Error( + "Invalid Firehose request", + zap.Error(errInHeaderMissingRequestID), + ) + fmr.sendResponse(w, requestID, http.StatusBadRequest, errInHeaderMissingRequestID) + return + } + fmr.settings.Logger.Debug("Processing Firehose request", zap.String("RequestID", requestID)) + + if statusCode, err := fmr.validate(r); err != nil { + fmr.settings.Logger.Error( + "Invalid Firehose request", + zap.Error(err), + ) + fmr.sendResponse(w, requestID, statusCode, err) + return + } + + body, err := fmr.getBody(r) + if err != nil { + fmr.sendResponse(w, requestID, http.StatusBadRequest, err) + return + } + + var fr firehoseRequest + if err = json.Unmarshal(body, &fr); err != nil { + fmr.sendResponse(w, requestID, http.StatusBadRequest, err) + return + } + + if fr.RequestID == "" { + fmr.sendResponse(w, requestID, http.StatusBadRequest, errInBodyMissingRequestID) + return + } else if fr.RequestID != requestID { + fmr.sendResponse(w, requestID, http.StatusBadRequest, errInBodyDiffRequestID) + return + } + + records := make([][]byte, 0, len(fr.Records)) + for index, record := range fr.Records { + if record.Data != "" { + var decoded []byte + decoded, err = base64.StdEncoding.DecodeString(record.Data) + if err != nil { + fmr.sendResponse( + w, + requestID, + http.StatusBadRequest, + fmt.Errorf("unable to base64 decode the record at index %d: %w", index, err), + ) + return + } + records = append(records, decoded) + } + } + + commonAttributes, err := fmr.getCommonAttributes(r) + if err != nil { + fmr.settings.Logger.Error( + "Unable to get common attributes from request header. Will not attach attributes.", + zap.Error(err), + ) + } + + statusCode, err := fmr.consumer.Consume(ctx, records, commonAttributes) + if err != nil { + fmr.settings.Logger.Error( + "Unable to consume records", + zap.Error(err), + ) + fmr.sendResponse(w, requestID, statusCode, err) + return + } + + fmr.sendResponse(w, requestID, http.StatusOK, nil) +} + +// validate checks the Firehose access key in the header against +// the one passed into the Config +func (fmr *firehoseReceiver) validate(r *http.Request) (int, error) { + if string(fmr.config.AccessKey) == "" { + // No access key is configured - accept all requests. + return http.StatusAccepted, nil + } + if accessKey := r.Header.Get(headerFirehoseAccessKey); accessKey == string(fmr.config.AccessKey) { + return http.StatusAccepted, nil + } + return http.StatusUnauthorized, errInvalidAccessKey +} + +// getBody reads the body from the request as a slice of bytes. +func (fmr *firehoseReceiver) getBody(r *http.Request) ([]byte, error) { + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + err = r.Body.Close() + if err != nil { + return nil, err + } + return body, nil +} + +// getCommonAttributes unmarshalls the common attributes from the request header +func (fmr *firehoseReceiver) getCommonAttributes(r *http.Request) (map[string]string, error) { + attributes := make(map[string]string) + if commonAttributes := r.Header.Get(headerFirehoseCommonAttributes); commonAttributes != "" { + var fca firehoseCommonAttributes + if err := json.Unmarshal([]byte(commonAttributes), &fca); err != nil { + return nil, err + } + attributes = fca.CommonAttributes + } + return attributes, nil +} + +// sendResponse writes a response to Firehose in the expected format. +func (fmr *firehoseReceiver) sendResponse(w http.ResponseWriter, requestID string, statusCode int, err error) { + var errorMessage string + if err != nil { + errorMessage = err.Error() + } + body := firehoseResponse{ + RequestID: requestID, + Timestamp: time.Now().UnixMilli(), + ErrorMessage: errorMessage, + } + payload, _ := json.Marshal(body) + w.Header().Set(headerContentType, "application/json") + w.Header().Set(headerContentLength, strconv.Itoa(len(payload))) + w.WriteHeader(statusCode) + if _, err = w.Write(payload); err != nil { + fmr.settings.Logger.Error("Failed to send response", zap.Error(err)) + } +} diff --git a/receiver/signozawsfirehosereceiver/receiver_test.go b/receiver/signozawsfirehosereceiver/receiver_test.go new file mode 100644 index 00000000..2a708464 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/receiver_test.go @@ -0,0 +1,252 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awsfirehosereceiver + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +const ( + testFirehoseRequestID = "firehose-request-id" + testFirehoseAccessKey = "firehose-access-key" +) + +type nopFirehoseConsumer struct { + statusCode int + err error +} + +var _ firehoseConsumer = (*nopFirehoseConsumer)(nil) + +func newNopFirehoseConsumer(statusCode int, err error) *nopFirehoseConsumer { + return &nopFirehoseConsumer{statusCode, err} +} + +func (nfc *nopFirehoseConsumer) Consume(context.Context, [][]byte, map[string]string) (int, error) { + return nfc.statusCode, nfc.err +} + +func TestStart(t *testing.T) { + testCases := map[string]struct { + host component.Host + wantErr error + }{ + "WithoutHost": { + wantErr: errMissingHost, + }, + "WithHost": { + host: componenttest.NewNopHost(), + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + cfg := &Config{} + ctx := context.TODO() + r := testFirehoseReceiver(cfg, nil) + got := r.Start(ctx, testCase.host) + require.Equal(t, testCase.wantErr, got) + if r.server != nil { + require.NoError(t, r.Shutdown(ctx)) + } + }) + } + t.Run("WithPortTaken", func(t *testing.T) { + listener, err := net.Listen("tcp", "localhost:") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, listener.Close()) + }) + cfg := &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: listener.Addr().String(), + }, + } + ctx := context.TODO() + r := testFirehoseReceiver(cfg, nil) + got := r.Start(ctx, componenttest.NewNopHost()) + require.Error(t, got) + if r.server != nil { + require.NoError(t, r.Shutdown(ctx)) + } + }) +} + +func TestFirehoseRequest(t *testing.T) { + defaultConsumer := newNopFirehoseConsumer(http.StatusOK, nil) + firehoseConsumerErr := errors.New("firehose consumer error") + cfg := &Config{ + AccessKey: testFirehoseAccessKey, + } + var noRecords []firehoseRecord + testCases := map[string]struct { + headers map[string]string + commonAttributes map[string]string + body any + consumer firehoseConsumer + wantStatusCode int + wantErr error + }{ + "WithoutRequestId/Header": { + headers: map[string]string{ + headerFirehoseRequestID: "", + }, + body: testFirehoseRequest(testFirehoseRequestID, noRecords), + wantStatusCode: http.StatusBadRequest, + wantErr: errInHeaderMissingRequestID, + }, + "WithDifferentAccessKey": { + headers: map[string]string{ + headerFirehoseAccessKey: "test", + }, + body: testFirehoseRequest(testFirehoseRequestID, noRecords), + wantStatusCode: http.StatusUnauthorized, + wantErr: errInvalidAccessKey, + }, + "WithNoAccessKey": { + headers: map[string]string{ + headerFirehoseAccessKey: "", + }, + body: testFirehoseRequest(testFirehoseRequestID, noRecords), + wantStatusCode: http.StatusUnauthorized, + wantErr: errInvalidAccessKey, + }, + "WithoutRequestId/Body": { + headers: map[string]string{ + headerFirehoseRequestID: testFirehoseRequestID, + }, + body: testFirehoseRequest("", noRecords), + wantStatusCode: http.StatusBadRequest, + wantErr: errInBodyMissingRequestID, + }, + "WithDifferentRequestIds": { + headers: map[string]string{ + headerFirehoseRequestID: testFirehoseRequestID, + }, + body: testFirehoseRequest("otherId", noRecords), + wantStatusCode: http.StatusBadRequest, + wantErr: errInBodyDiffRequestID, + }, + "WithInvalidBody": { + body: "{ test: ", + wantStatusCode: http.StatusBadRequest, + wantErr: errors.New("json: cannot unmarshal string into Go value of type awsfirehosereceiver.firehoseRequest"), + }, + "WithNoRecords": { + body: testFirehoseRequest(testFirehoseRequestID, noRecords), + wantStatusCode: http.StatusOK, + }, + "WithFirehoseConsumerError": { + body: testFirehoseRequest(testFirehoseRequestID, noRecords), + consumer: newNopFirehoseConsumer(http.StatusInternalServerError, firehoseConsumerErr), + wantStatusCode: http.StatusInternalServerError, + wantErr: firehoseConsumerErr, + }, + "WithCorruptBase64Records": { + body: testFirehoseRequest(testFirehoseRequestID, []firehoseRecord{ + {Data: "XXXXXaGVsbG8="}, + }), + wantStatusCode: http.StatusBadRequest, + wantErr: fmt.Errorf("unable to base64 decode the record at index 0: %w", base64.CorruptInputError(12)), + }, + "WithValidRecords": { + body: testFirehoseRequest(testFirehoseRequestID, []firehoseRecord{ + testFirehoseRecord("test"), + }), + wantStatusCode: http.StatusOK, + }, + "WithValidRecords/CommonAttributes": { + body: testFirehoseRequest(testFirehoseRequestID, []firehoseRecord{ + testFirehoseRecord("test"), + }), + commonAttributes: map[string]string{ + "TestAttribute": "common", + }, + wantStatusCode: http.StatusOK, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + body, err := json.Marshal(testCase.body) + require.NoError(t, err) + + requestBody := bytes.NewBuffer(body) + + request := httptest.NewRequest(http.MethodPost, "/", requestBody) + request.Header.Set(headerContentType, "application/json") + request.Header.Set(headerContentLength, strconv.Itoa(requestBody.Len())) + request.Header.Set(headerFirehoseRequestID, testFirehoseRequestID) + request.Header.Set(headerFirehoseAccessKey, testFirehoseAccessKey) + if testCase.headers != nil { + for k, v := range testCase.headers { + request.Header.Set(k, v) + } + } + if testCase.commonAttributes != nil { + attrs, err := json.Marshal(firehoseCommonAttributes{ + CommonAttributes: testCase.commonAttributes, + }) + require.NoError(t, err) + request.Header.Set(headerFirehoseCommonAttributes, string(attrs)) + } + + consumer := testCase.consumer + if consumer == nil { + consumer = defaultConsumer + } + r := testFirehoseReceiver(cfg, consumer) + + got := httptest.NewRecorder() + r.ServeHTTP(got, request) + + require.Equal(t, testCase.wantStatusCode, got.Code) + var gotResponse firehoseResponse + require.NoError(t, json.Unmarshal(got.Body.Bytes(), &gotResponse)) + require.Equal(t, request.Header.Get(headerFirehoseRequestID), gotResponse.RequestID) + if testCase.wantErr != nil { + require.Equal(t, testCase.wantErr.Error(), gotResponse.ErrorMessage) + } else { + require.Empty(t, gotResponse.ErrorMessage) + } + }) + } +} + +// testFirehoseReceiver is a convenience function for creating a test firehoseReceiver +func testFirehoseReceiver(config *Config, consumer firehoseConsumer) *firehoseReceiver { + return &firehoseReceiver{ + settings: receivertest.NewNopSettings(), + config: config, + consumer: consumer, + } +} + +func testFirehoseRequest(requestID string, records []firehoseRecord) firehoseRequest { + return firehoseRequest{ + RequestID: requestID, + Timestamp: time.Now().UnixMilli(), + Records: records, + } +} + +func testFirehoseRecord(data string) firehoseRecord { + encoded := base64.StdEncoding.EncodeToString([]byte(data)) + return firehoseRecord{Data: encoded} +} diff --git a/receiver/signozawsfirehosereceiver/testdata/cwlogs_config.yaml b/receiver/signozawsfirehosereceiver/testdata/cwlogs_config.yaml new file mode 100644 index 00000000..9ec04ad0 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/testdata/cwlogs_config.yaml @@ -0,0 +1,7 @@ +awsfirehose: + endpoint: 0.0.0.0:4433 + record_type: cwlogs + access_key: "some_access_key" + tls: + cert_file: server.crt + key_file: server.key diff --git a/receiver/signozawsfirehosereceiver/testdata/cwmetrics_config.yaml b/receiver/signozawsfirehosereceiver/testdata/cwmetrics_config.yaml new file mode 100644 index 00000000..13dbc9fe --- /dev/null +++ b/receiver/signozawsfirehosereceiver/testdata/cwmetrics_config.yaml @@ -0,0 +1,7 @@ +awsfirehose: + endpoint: 0.0.0.0:4433 + record_type: cwmetrics + access_key: "some_access_key" + tls: + cert_file: server.crt + key_file: server.key diff --git a/receiver/signozawsfirehosereceiver/testdata/invalid_config.yaml b/receiver/signozawsfirehosereceiver/testdata/invalid_config.yaml new file mode 100644 index 00000000..17ad1316 --- /dev/null +++ b/receiver/signozawsfirehosereceiver/testdata/invalid_config.yaml @@ -0,0 +1,3 @@ +awsfirehose: + endpoint: 0.0.0.0:4433 + record_type: invalid \ No newline at end of file diff --git a/receiver/signozawsfirehosereceiver/testdata/otlp_v1_config.yaml b/receiver/signozawsfirehosereceiver/testdata/otlp_v1_config.yaml new file mode 100644 index 00000000..fe9def7f --- /dev/null +++ b/receiver/signozawsfirehosereceiver/testdata/otlp_v1_config.yaml @@ -0,0 +1,7 @@ +awsfirehose: + endpoint: 0.0.0.0:4433 + record_type: otlp_v1 + access_key: "some_access_key" + tls: + cert_file: server.crt + key_file: server.key