From d642ab0071de78c5c53cbb33682a400eaea05220 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Wed, 7 Apr 2021 16:20:21 -0700 Subject: [PATCH 01/12] added telemetry support --- base_transport.go | 2 +- client.go | 41 +++++++++++++--- queue.go | 59 +++++++++++++++++++++++ rollbar.go | 16 +++++++ telemetry.go | 116 ++++++++++++++++++++++++++++++++++++++++++++++ transforms.go | 8 ++-- 6 files changed, 232 insertions(+), 10 deletions(-) create mode 100644 queue.go create mode 100644 telemetry.go diff --git a/base_transport.go b/base_transport.go index 0b16ef4..50d8dd1 100644 --- a/base_transport.go +++ b/base_transport.go @@ -68,7 +68,7 @@ func (t *baseTransport) getHTTPClient() *http.Client { return t.httpClient } - return http.DefaultClient + return &http.Client{} } // post returns an error which indicates the type of error that occurred while attempting to diff --git a/client.go b/client.go index f9d966e..b9ea7e2 100644 --- a/client.go +++ b/client.go @@ -10,6 +10,7 @@ import ( "reflect" "regexp" "runtime" + "time" ) // A Client can be used to interact with Rollbar via the configured Transport. @@ -23,6 +24,7 @@ type Client struct { // Transport used to send data to the Rollbar API. By default an asynchronous // implementation of the Transport interface is used. Transport Transport + Telemetry *Telemetry configuration configuration diagnostic diagnostic } @@ -40,6 +42,7 @@ func NewAsync(token, environment, codeVersion, serverHost, serverRoot string) *C diagnostic := createDiagnostic() return &Client{ Transport: transport, + Telemetry: NewTelemetry(), configuration: configuration, diagnostic: diagnostic, } @@ -52,11 +55,29 @@ func NewSync(token, environment, codeVersion, serverHost, serverRoot string) *Cl diagnostic := createDiagnostic() return &Client{ Transport: transport, + Telemetry: NewTelemetry(), configuration: configuration, diagnostic: diagnostic, } } +// CaptureTelemetryEvent sets the user-specified telemetry event +func (c *Client) CaptureTelemetryEvent(eventType, eventlevel string, eventData map[string]interface{}) { + data := map[string]interface{}{} + data["body"] = eventData + data["type"] = eventType + data["level"] = eventlevel + data["source"] = "client" + data["timestamp_ms"] = time.Now().UnixNano() / int64(time.Millisecond) + + c.Telemetry.Queue.Push(data) +} + +// SetTelemetry sets the telemetry +func (c *Client) SetTelemetry(t *Telemetry) { + c.Telemetry = t +} + // SetEnabled sets whether or not Rollbar is enabled. // If this is true then this library works as normal. // If this is false then no calls will be made to the network. @@ -361,7 +382,8 @@ func (c *Client) ErrorWithStackSkipWithExtrasAndContext(ctx context.Context, lev return } body := c.buildBody(ctx, level, err.Error(), extras) - addErrorToBody(c.configuration, body, err, skip) + telemetry := c.Telemetry.GetQueueItems() + addErrorToBody(c.configuration, body, err, skip, telemetry) c.push(body) } @@ -389,7 +411,8 @@ func (c *Client) RequestErrorWithStackSkipWithExtrasAndContext(ctx context.Conte return } body := c.buildBody(ctx, level, err.Error(), extras) - data := addErrorToBody(c.configuration, body, err, skip) + telemetry := c.Telemetry.GetQueueItems() + data := addErrorToBody(c.configuration, body, err, skip, telemetry) data["request"] = c.requestDetails(r) c.push(body) } @@ -415,7 +438,10 @@ func (c *Client) MessageWithExtrasAndContext(ctx context.Context, level string, } body := c.buildBody(ctx, level, msg, extras) data := body["data"].(map[string]interface{}) - data["body"] = messageBody(msg) + dataBody := messageBody(msg) + telemetry := c.Telemetry.GetQueueItems() + dataBody["telemetry"] = telemetry + data["body"] = dataBody c.push(body) } @@ -440,7 +466,10 @@ func (c *Client) RequestMessageWithExtrasAndContext(ctx context.Context, level s } body := c.buildBody(ctx, level, msg, extras) data := body["data"].(map[string]interface{}) - data["body"] = messageBody(msg) + dataBody := messageBody(msg) + telemetry := c.Telemetry.GetQueueItems() + dataBody["telemetry"] = telemetry + data["body"] = dataBody data["request"] = c.requestDetails(r) c.push(body) } @@ -669,12 +698,12 @@ func createConfiguration(token, environment, codeVersion, serverHost, serverRoot } type diagnostic struct { - languageVersion string + languageVersion string } func createDiagnostic() diagnostic { return diagnostic{ - languageVersion: runtime.Version(), + languageVersion: runtime.Version(), } } diff --git a/queue.go b/queue.go new file mode 100644 index 0000000..9cc0923 --- /dev/null +++ b/queue.go @@ -0,0 +1,59 @@ +package rollbar + +import "sync" + +// NewQueue returns a new queue with the given initial size. +func NewQueue(size int) *Queue { + return &Queue{ + nodes: make([]interface{}, size), + size: size, + } +} + +// Queue is a basic FIFO queue based on a circular list that resizes as needed. +type Queue struct { + nodes []interface{} + size int + head int + tail int + count int + + lock sync.RWMutex +} + +// Push adds a node to the queue. +func (q *Queue) Push(n interface{}) { + q.lock.Lock() + defer q.lock.Unlock() + if q.head == q.tail && q.count > 0 { + nodes := make([]interface{}, len(q.nodes)+q.size) + copy(nodes, q.nodes[q.head:]) + copy(nodes[len(q.nodes)-q.head:], q.nodes[:q.head]) + q.head = 0 + q.tail = len(q.nodes) + q.nodes = nodes + } + q.nodes[q.tail] = n + q.tail = (q.tail + 1) % len(q.nodes) + q.count++ +} + +// Pop removes and returns a node from the queue in first to last order. +func (q *Queue) Pop() interface{} { + q.lock.Lock() + defer q.lock.Unlock() + if q.count == 0 { + return nil + } + node := q.nodes[q.head] + q.head = (q.head + 1) % len(q.nodes) + q.count-- + return node +} + +// Items returns all populated (non nil) items +func (q *Queue) Items() []interface{} { + q.lock.RLock() + defer q.lock.RUnlock() + return q.nodes[:q.count] +} diff --git a/rollbar.go b/rollbar.go index 860fc47..81c89b7 100644 --- a/rollbar.go +++ b/rollbar.go @@ -90,6 +90,22 @@ var DefaultStackTracer StackTracerFunc = func(err error) ([]runtime.Frame, bool) return nil, false } +// GetTelemetryHTTPClientTransport enables a user to set Transport on http client: +// example: client := &http.Client{Transport: rollbar.GetTelemetryHTTPClientTransport()} +func GetTelemetryHTTPClientTransport() http.RoundTripper { + return std.Telemetry +} + +// SetTelemetry sets the telemetry +func SetTelemetry(t *Telemetry) { + std.SetTelemetry(t) +} + +// CaptureTelemetryEvent sets the user-specified telemetry event +func CaptureTelemetryEvent(eventType, eventlevel string, eventData map[string]interface{}) { + std.CaptureTelemetryEvent(eventType, eventlevel, eventData) +} + // SetEnabled sets whether or not the managed Client instance is enabled. // If this is true then this library works as normal. // If this is false then no calls will be made to the network. diff --git a/telemetry.go b/telemetry.go new file mode 100644 index 0000000..318c70f --- /dev/null +++ b/telemetry.go @@ -0,0 +1,116 @@ +package rollbar + +import ( + "fmt" + "io" + "log" + "net/http" + "os" + "time" +) + +// Telemetry struct contains writer (for logs) and round tripper (for http client) and enables to queue the events +type Telemetry struct { + Writer io.Writer + Proxied http.RoundTripper + Queue *Queue +} + +// Write is the writer for telemetry logs +func (t *Telemetry) Write(p []byte) (int, error) { + telemetryData := t.populateLoggerBody(p) + t.Queue.Push(telemetryData) + return t.Writer.Write(p) +} + +// RoundTrip implements RoundTrip in http.RoundTripper +func (t *Telemetry) RoundTrip(req *http.Request) (res *http.Response, e error) { + + // Send the request, get the response (or the error) + res, e = t.Proxied.RoundTrip(req) + if e != nil { + fmt.Printf("Error: %v", e) + } + telemetryData := t.populateTransporterBody(req, res) + t.Queue.Push(telemetryData) + return +} + +func (t *Telemetry) populateLoggerBody(p []byte) map[string]interface{} { + var data = map[string]interface{}{} + message := map[string]interface{}{"message": string(p)} + data["body"] = message + data["source"] = "client" + data["timestamp_ms"] = time.Now().UnixNano() / int64(time.Millisecond) + data["type"] = "log" + data["level"] = "log" + return data +} + +func (t *Telemetry) populateTransporterBody(req *http.Request, res *http.Response) map[string]interface{} { + var data = map[string]interface{}{} + var dataBody = map[string]interface{}{} + var dataHeaders = map[string][]string{} + dataBody["status_code"] = nil + data["level"] = "info" + if res != nil { + dataBody["status_code"] = res.StatusCode + if res.StatusCode >= http.StatusInternalServerError { + data["level"] = "critical" + } else if res.StatusCode >= http.StatusBadRequest { + data["level"] = "error" + } + } + dataBody["url"] = req.URL.Scheme + "://" + req.Host + req.URL.Path + dataBody["method"] = req.Method + dataBody["subtype"] = "http" + + for k, v := range req.Header { + dataHeaders[k] = v + } + dataBody["request_headers"] = dataHeaders + + data["body"] = dataBody + data["source"] = "client" + data["timestamp_ms"] = time.Now().UnixNano() / int64(time.Millisecond) + data["type"] = "network" + return data +} + +// GetQueueItems gets all the items from the queue +func (t *Telemetry) GetQueueItems() []interface{} { + return t.Queue.Items() +} + +// OptionFunc is the pointer to the optional parameter function +type OptionFunc func(*Telemetry) + +// WithCustomTransporter sets the custom transporter +func WithCustomTransporter(t http.RoundTripper) OptionFunc { + return func(f *Telemetry) { + f.Proxied = t + } +} + +// WithCustomQueueSize initializes the queue with a custom size +func WithCustomQueueSize(size int) OptionFunc { + return func(f *Telemetry) { + f.Queue = NewQueue(size) + } +} + +// NewTelemetry initializes telemetry object +func NewTelemetry(options ...OptionFunc) *Telemetry { + res := &Telemetry{ + Proxied: http.DefaultTransport, + Queue: NewQueue(50), + Writer: os.Stdout, + } + for _, opt := range options { + opt(res) + } + + log.SetOutput(res) + http.DefaultClient = &http.Client{Transport: res} + return res +} diff --git a/transforms.go b/transforms.go index e5339ff..11278e5 100644 --- a/transforms.go +++ b/transforms.go @@ -34,7 +34,7 @@ func buildBody(ctx context.Context, configuration configuration, diagnostic diag "name": NAME, "version": VERSION, "diagnostic": map[string]interface{}{ - "languageVersion": diagnostic.languageVersion, + "languageVersion": diagnostic.languageVersion, "configuredOptions": buildConfiguredOptions(configuration), }, }, @@ -101,10 +101,12 @@ func buildConfiguredOptions(configuration configuration) map[string]interface{} } } -func addErrorToBody(configuration configuration, body map[string]interface{}, err error, skip int) map[string]interface{} { +func addErrorToBody(configuration configuration, body map[string]interface{}, err error, skip int, telemetry []interface{}) map[string]interface{} { data := body["data"].(map[string]interface{}) errBody, fingerprint := errorBody(configuration, err, skip) - data["body"] = errBody + dataBody := errBody + dataBody["telemetry"] = telemetry + data["body"] = dataBody if configuration.fingerprint { data["fingerprint"] = fingerprint } From e55447e5af6a96bcb61b8d1aaa4f99a2bd33b02f Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 12 Apr 2021 13:35:24 -0700 Subject: [PATCH 02/12] addressing comments after review --- client.go | 5 ++- rollbar.go | 12 +++--- telemetry.go | 103 ++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/client.go b/client.go index b9ea7e2..b083d0d 100644 --- a/client.go +++ b/client.go @@ -74,8 +74,8 @@ func (c *Client) CaptureTelemetryEvent(eventType, eventlevel string, eventData m } // SetTelemetry sets the telemetry -func (c *Client) SetTelemetry(t *Telemetry) { - c.Telemetry = t +func (c *Client) SetTelemetry(options ...OptionFunc) { + c.Telemetry = NewTelemetry(options...) } // SetEnabled sets whether or not Rollbar is enabled. @@ -168,6 +168,7 @@ func (c *Client) SetLogger(logger ClientLogger) { // The default value is regexp.MustCompile("Authorization") func (c *Client) SetScrubHeaders(headers *regexp.Regexp) { c.configuration.scrubHeaders = headers + c.Telemetry.Network.ScrubHeaders = headers } // SetScrubFields sets the regular expression to match keys in the item payload for scrubbing. diff --git a/rollbar.go b/rollbar.go index 81c89b7..d726e76 100644 --- a/rollbar.go +++ b/rollbar.go @@ -90,15 +90,15 @@ var DefaultStackTracer StackTracerFunc = func(err error) ([]runtime.Frame, bool) return nil, false } -// GetTelemetryHTTPClientTransport enables a user to set Transport on http client: -// example: client := &http.Client{Transport: rollbar.GetTelemetryHTTPClientTransport()} -func GetTelemetryHTTPClientTransport() http.RoundTripper { - return std.Telemetry +// SetHTTPClientForTelemetry sets the http client for telemetry. +// client is passed by the reference and it's sets the Transport on the client. +func SetHTTPClientForTelemetry(httpClient *http.Client) { + httpClient.Transport = std.Telemetry } // SetTelemetry sets the telemetry -func SetTelemetry(t *Telemetry) { - std.SetTelemetry(t) +func SetTelemetry(options ...OptionFunc) { + std.SetTelemetry(options...) } // CaptureTelemetryEvent sets the user-specified telemetry event diff --git a/telemetry.go b/telemetry.go index 318c70f..2fd988b 100644 --- a/telemetry.go +++ b/telemetry.go @@ -6,28 +6,41 @@ import ( "log" "net/http" "os" + "regexp" "time" ) +const TelemetryQueueSize = 50 + // Telemetry struct contains writer (for logs) and round tripper (for http client) and enables to queue the events type Telemetry struct { - Writer io.Writer - Proxied http.RoundTripper - Queue *Queue + Logger struct { + Writer io.Writer + } + + Network struct { + Proxied http.RoundTripper + ScrubHeaders *regexp.Regexp + + enbaleDefaultClient bool + disableReqHeaders bool + disableResHeaders bool + } + Queue *Queue } // Write is the writer for telemetry logs func (t *Telemetry) Write(p []byte) (int, error) { telemetryData := t.populateLoggerBody(p) t.Queue.Push(telemetryData) - return t.Writer.Write(p) + return t.Logger.Writer.Write(p) } // RoundTrip implements RoundTrip in http.RoundTripper func (t *Telemetry) RoundTrip(req *http.Request) (res *http.Response, e error) { // Send the request, get the response (or the error) - res, e = t.Proxied.RoundTrip(req) + res, e = t.Network.Proxied.RoundTrip(req) if e != nil { fmt.Printf("Error: %v", e) } @@ -50,7 +63,6 @@ func (t *Telemetry) populateLoggerBody(p []byte) map[string]interface{} { func (t *Telemetry) populateTransporterBody(req *http.Request, res *http.Response) map[string]interface{} { var data = map[string]interface{}{} var dataBody = map[string]interface{}{} - var dataHeaders = map[string][]string{} dataBody["status_code"] = nil data["level"] = "info" if res != nil { @@ -60,16 +72,30 @@ func (t *Telemetry) populateTransporterBody(req *http.Request, res *http.Respons } else if res.StatusCode >= http.StatusBadRequest { data["level"] = "error" } + + if !t.Network.disableResHeaders { + var dataHeaders = map[string][]string{} + for k, v := range res.Header { + dataHeaders[k] = v + } + filteredDataHeaders := filterFlatten(t.Network.ScrubHeaders, dataHeaders, nil) + response := map[string]interface{}{"headers": filteredDataHeaders} + dataBody["response"] = response + } + } dataBody["url"] = req.URL.Scheme + "://" + req.Host + req.URL.Path dataBody["method"] = req.Method dataBody["subtype"] = "http" - for k, v := range req.Header { - dataHeaders[k] = v + if !t.Network.disableReqHeaders { + var dataHeaders = map[string][]string{} + for k, v := range req.Header { + dataHeaders[k] = v + } + filteredDataHeaders := filterFlatten(t.Network.ScrubHeaders, dataHeaders, nil) + dataBody["request_headers"] = filteredDataHeaders } - dataBody["request_headers"] = dataHeaders - data["body"] = dataBody data["source"] = "client" data["timestamp_ms"] = time.Now().UnixNano() / int64(time.Millisecond) @@ -85,32 +111,67 @@ func (t *Telemetry) GetQueueItems() []interface{} { // OptionFunc is the pointer to the optional parameter function type OptionFunc func(*Telemetry) -// WithCustomTransporter sets the custom transporter -func WithCustomTransporter(t http.RoundTripper) OptionFunc { +// EnableNetworkTelemetry enables the network telemetry. +// if no custom http Transport is needed, then nil can be passed +func EnableNetworkTelemetry(t http.RoundTripper) OptionFunc { + return func(f *Telemetry) { + f.Network.Proxied = http.DefaultTransport + if t != nil { + f.Network.Proxied = t + } + } +} + +// EnableNetworkTelemetryForDefaultClient sets the http.DefaultClient for telemetry +func EnableNetworkTelemetryForDefaultClient() OptionFunc { return func(f *Telemetry) { - f.Proxied = t + f.Network.enbaleDefaultClient = true } } -// WithCustomQueueSize initializes the queue with a custom size -func WithCustomQueueSize(size int) OptionFunc { +// DisableNetworkTelemetryRequestHeaders disables telemetry request headers +func DisableNetworkTelemetryRequestHeaders() OptionFunc { + return func(f *Telemetry) { + f.Network.disableReqHeaders = true + } +} + +// DisableNetworkTelemetryResponseHeaders disables telemetry response headers +func DisableNetworkTelemetryResponseHeaders() OptionFunc { + return func(f *Telemetry) { + f.Network.disableResHeaders = true + } +} + +// SetCustomQueueSize initializes the queue with a custom size +func SetCustomQueueSize(size int) OptionFunc { return func(f *Telemetry) { f.Queue = NewQueue(size) } } +// EnableLoggerTelemetry enables logger telemetry +func EnableLoggerTelemetry() OptionFunc { + return func(f *Telemetry) { + f.Logger.Writer = os.Stdout + log.SetOutput(f) + } +} + // NewTelemetry initializes telemetry object func NewTelemetry(options ...OptionFunc) *Telemetry { res := &Telemetry{ - Proxied: http.DefaultTransport, - Queue: NewQueue(50), - Writer: os.Stdout, + Queue: NewQueue(TelemetryQueueSize), } + for _, opt := range options { opt(res) } - - log.SetOutput(res) - http.DefaultClient = &http.Client{Transport: res} + if res.Network.enbaleDefaultClient { + http.DefaultClient.Transport = res + } + if res.Network.ScrubHeaders == nil { // set/define only once + res.Network.ScrubHeaders = regexp.MustCompile("Authorization") + } return res } From 4ae7153e6dfe11bf486528b22466b55ff40581ca Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 12 Apr 2021 13:43:16 -0700 Subject: [PATCH 03/12] order of setting is important here --- telemetry.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/telemetry.go b/telemetry.go index 2fd988b..d6e9a83 100644 --- a/telemetry.go +++ b/telemetry.go @@ -167,11 +167,12 @@ func NewTelemetry(options ...OptionFunc) *Telemetry { for _, opt := range options { opt(res) } - if res.Network.enbaleDefaultClient { - http.DefaultClient.Transport = res - } + if res.Network.ScrubHeaders == nil { // set/define only once res.Network.ScrubHeaders = regexp.MustCompile("Authorization") } + if res.Network.enbaleDefaultClient { + http.DefaultClient.Transport = res + } return res } From 2cb6083e4bfa39e73eab4a202f2863d5b3e0f12e Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Tue, 13 Apr 2021 16:17:45 -0700 Subject: [PATCH 04/12] addressing comments after review --- base_transport.go | 2 +- telemetry.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base_transport.go b/base_transport.go index 50d8dd1..0b16ef4 100644 --- a/base_transport.go +++ b/base_transport.go @@ -68,7 +68,7 @@ func (t *baseTransport) getHTTPClient() *http.Client { return t.httpClient } - return &http.Client{} + return http.DefaultClient } // post returns an error which indicates the type of error that occurred while attempting to diff --git a/telemetry.go b/telemetry.go index d6e9a83..9762303 100644 --- a/telemetry.go +++ b/telemetry.go @@ -22,7 +22,7 @@ type Telemetry struct { Proxied http.RoundTripper ScrubHeaders *regexp.Regexp - enbaleDefaultClient bool + enableDefaultClient bool disableReqHeaders bool disableResHeaders bool } @@ -125,7 +125,7 @@ func EnableNetworkTelemetry(t http.RoundTripper) OptionFunc { // EnableNetworkTelemetryForDefaultClient sets the http.DefaultClient for telemetry func EnableNetworkTelemetryForDefaultClient() OptionFunc { return func(f *Telemetry) { - f.Network.enbaleDefaultClient = true + f.Network.enableDefaultClient = true } } @@ -171,7 +171,7 @@ func NewTelemetry(options ...OptionFunc) *Telemetry { if res.Network.ScrubHeaders == nil { // set/define only once res.Network.ScrubHeaders = regexp.MustCompile("Authorization") } - if res.Network.enbaleDefaultClient { + if res.Network.enableDefaultClient { http.DefaultClient.Transport = res } return res From 4910d978ea9e3ee030f2b80c707ace8dd48aff6d Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Wed, 14 Apr 2021 12:03:18 -0700 Subject: [PATCH 05/12] addressing PR comments --- rollbar.go | 6 ------ telemetry.go | 33 +++++++++++++++++---------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/rollbar.go b/rollbar.go index d726e76..33776be 100644 --- a/rollbar.go +++ b/rollbar.go @@ -90,12 +90,6 @@ var DefaultStackTracer StackTracerFunc = func(err error) ([]runtime.Frame, bool) return nil, false } -// SetHTTPClientForTelemetry sets the http client for telemetry. -// client is passed by the reference and it's sets the Transport on the client. -func SetHTTPClientForTelemetry(httpClient *http.Client) { - httpClient.Transport = std.Telemetry -} - // SetTelemetry sets the telemetry func SetTelemetry(options ...OptionFunc) { std.SetTelemetry(options...) diff --git a/telemetry.go b/telemetry.go index 9762303..f42c197 100644 --- a/telemetry.go +++ b/telemetry.go @@ -23,8 +23,8 @@ type Telemetry struct { ScrubHeaders *regexp.Regexp enableDefaultClient bool - disableReqHeaders bool - disableResHeaders bool + enableReqHeaders bool + enableResHeaders bool } Queue *Queue } @@ -73,7 +73,7 @@ func (t *Telemetry) populateTransporterBody(req *http.Request, res *http.Respons data["level"] = "error" } - if !t.Network.disableResHeaders { + if t.Network.enableResHeaders { var dataHeaders = map[string][]string{} for k, v := range res.Header { dataHeaders[k] = v @@ -88,7 +88,7 @@ func (t *Telemetry) populateTransporterBody(req *http.Request, res *http.Respons dataBody["method"] = req.Method dataBody["subtype"] = "http" - if !t.Network.disableReqHeaders { + if t.Network.enableReqHeaders { var dataHeaders = map[string][]string{} for k, v := range req.Header { dataHeaders[k] = v @@ -112,13 +112,11 @@ func (t *Telemetry) GetQueueItems() []interface{} { type OptionFunc func(*Telemetry) // EnableNetworkTelemetry enables the network telemetry. -// if no custom http Transport is needed, then nil can be passed -func EnableNetworkTelemetry(t http.RoundTripper) OptionFunc { +// it wraps up the client for telemetry +func EnableNetworkTelemetry(httpClient *http.Client) OptionFunc { return func(f *Telemetry) { - f.Network.Proxied = http.DefaultTransport - if t != nil { - f.Network.Proxied = t - } + f.Network.Proxied = httpClient.Transport + httpClient.Transport = f } } @@ -129,17 +127,17 @@ func EnableNetworkTelemetryForDefaultClient() OptionFunc { } } -// DisableNetworkTelemetryRequestHeaders disables telemetry request headers -func DisableNetworkTelemetryRequestHeaders() OptionFunc { +// EnableNetworkTelemetryRequestHeaders enables telemetry request headers +func EnableNetworkTelemetryRequestHeaders() OptionFunc { return func(f *Telemetry) { - f.Network.disableReqHeaders = true + f.Network.enableReqHeaders = true } } -// DisableNetworkTelemetryResponseHeaders disables telemetry response headers -func DisableNetworkTelemetryResponseHeaders() OptionFunc { +// EnableNetworkTelemetryResponseHeaders enables telemetry response headers +func EnableNetworkTelemetryResponseHeaders() OptionFunc { return func(f *Telemetry) { - f.Network.disableResHeaders = true + f.Network.enableResHeaders = true } } @@ -172,6 +170,9 @@ func NewTelemetry(options ...OptionFunc) *Telemetry { res.Network.ScrubHeaders = regexp.MustCompile("Authorization") } if res.Network.enableDefaultClient { + if res.Network.Proxied == nil { + res.Network.Proxied = http.DefaultTransport + } http.DefaultClient.Transport = res } return res From aa9e6012fbe660b568310f81aea5e43f16d6591b Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Wed, 14 Apr 2021 13:32:21 -0700 Subject: [PATCH 06/12] addressing PR comments --- telemetry.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/telemetry.go b/telemetry.go index f42c197..845c114 100644 --- a/telemetry.go +++ b/telemetry.go @@ -22,9 +22,8 @@ type Telemetry struct { Proxied http.RoundTripper ScrubHeaders *regexp.Regexp - enableDefaultClient bool - enableReqHeaders bool - enableResHeaders bool + enableReqHeaders bool + enableResHeaders bool } Queue *Queue } @@ -82,7 +81,6 @@ func (t *Telemetry) populateTransporterBody(req *http.Request, res *http.Respons response := map[string]interface{}{"headers": filteredDataHeaders} dataBody["response"] = response } - } dataBody["url"] = req.URL.Scheme + "://" + req.Host + req.URL.Path dataBody["method"] = req.Method @@ -111,8 +109,9 @@ func (t *Telemetry) GetQueueItems() []interface{} { // OptionFunc is the pointer to the optional parameter function type OptionFunc func(*Telemetry) -// EnableNetworkTelemetry enables the network telemetry. +// EnableNetworkTelemetry enables the network telemetry // it wraps up the client for telemetry +// http.DefaultClient can also be passed by the reference func EnableNetworkTelemetry(httpClient *http.Client) OptionFunc { return func(f *Telemetry) { f.Network.Proxied = httpClient.Transport @@ -120,13 +119,6 @@ func EnableNetworkTelemetry(httpClient *http.Client) OptionFunc { } } -// EnableNetworkTelemetryForDefaultClient sets the http.DefaultClient for telemetry -func EnableNetworkTelemetryForDefaultClient() OptionFunc { - return func(f *Telemetry) { - f.Network.enableDefaultClient = true - } -} - // EnableNetworkTelemetryRequestHeaders enables telemetry request headers func EnableNetworkTelemetryRequestHeaders() OptionFunc { return func(f *Telemetry) { @@ -169,11 +161,6 @@ func NewTelemetry(options ...OptionFunc) *Telemetry { if res.Network.ScrubHeaders == nil { // set/define only once res.Network.ScrubHeaders = regexp.MustCompile("Authorization") } - if res.Network.enableDefaultClient { - if res.Network.Proxied == nil { - res.Network.Proxied = http.DefaultTransport - } - http.DefaultClient.Transport = res - } + return res } From 758c440ed1652ec0bdea301207bada7e6fd1592b Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 16 Apr 2021 17:06:40 -0700 Subject: [PATCH 07/12] added initial tests --- go.mod | 2 + go.sum | 10 +++++ telemetry.go | 6 ++- telemetry_test.go | 106 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 telemetry_test.go diff --git a/go.mod b/go.mod index ac50a32..06412f5 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/rollbar/rollbar-go go 1.13 + +require github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index e69de29..b380ae4 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/telemetry.go b/telemetry.go index 845c114..7df8f64 100644 --- a/telemetry.go +++ b/telemetry.go @@ -114,7 +114,11 @@ type OptionFunc func(*Telemetry) // http.DefaultClient can also be passed by the reference func EnableNetworkTelemetry(httpClient *http.Client) OptionFunc { return func(f *Telemetry) { - f.Network.Proxied = httpClient.Transport + if httpClient.Transport == nil { + f.Network.Proxied = http.DefaultTransport + } else { + f.Network.Proxied = httpClient.Transport + } httpClient.Transport = f } } diff --git a/telemetry_test.go b/telemetry_test.go new file mode 100644 index 0000000..40a72ef --- /dev/null +++ b/telemetry_test.go @@ -0,0 +1,106 @@ +package rollbar + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "os" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewTelemetryDefault(t *testing.T) { + telemetry := NewTelemetry() + assert.NotNil(t, telemetry) + expectedTelemetry := &Telemetry{Queue: NewQueue(TelemetryQueueSize)} + expectedTelemetry.Network.ScrubHeaders = regexp.MustCompile("Authorization") + + assert.Equal(t, expectedTelemetry, telemetry) + +} + +func TestNewTelemetryWithOptions(t *testing.T) { + client := http.Client{} + telemetry := NewTelemetry(SetCustomQueueSize(100), EnableNetworkTelemetry(&client), + EnableNetworkTelemetryRequestHeaders(), EnableNetworkTelemetryResponseHeaders(), EnableLoggerTelemetry()) + expectedTelemetry := &Telemetry{Queue: telemetry.Queue} + expectedTelemetry.Network.ScrubHeaders = regexp.MustCompile("Authorization") + expectedTelemetry.Network.enableReqHeaders = true + expectedTelemetry.Network.enableResHeaders = true + expectedTelemetry.Network.Proxied = http.DefaultTransport + expectedTelemetry.Logger.Writer = os.Stdout + + assert.Equal(t, expectedTelemetry, telemetry) + assert.Equal(t, client.Transport, expectedTelemetry) +} +func TestPopulateBody(t *testing.T) { + req := httptest.NewRequest("GET", "/some_url", nil) + req.Header.Set("Some_name", "some_value") + rec := httptest.NewRecorder() + + telemetry := NewTelemetry() + EnableNetworkTelemetryRequestHeaders()(telemetry) + EnableNetworkTelemetryResponseHeaders()(telemetry) + data := telemetry.populateTransporterBody(req, rec.Result()) + assert.NotNil(t, data) + assert.True(t, data["timestamp_ms"].(int64) > 0) + delete(data, "timestamp_ms") + + expectedBodyData := map[string]interface{}{"method": "GET", "status_code": 200, "subtype": "http", + "url": "://example.com/some_url", "request_headers": map[string]interface{}{"Some_name": "some_value"}, + "response": map[string]interface{}{"headers": map[string]interface{}{}}} + + expectedData := map[string]interface{}{"body": expectedBodyData, "level": "info", "source": "client", "type": "network"} + + assert.Equal(t, expectedData, data) +} + +func TestPopulateLoggerBody(t *testing.T) { + + message := "some message" + telemetry := NewTelemetry() + + data := telemetry.populateLoggerBody([]byte(message)) + + assert.NotNil(t, data) + assert.True(t, data["timestamp_ms"].(int64) > 0) + delete(data, "timestamp_ms") + expectedData := map[string]interface{}{"body": map[string]interface{}{"message": message}, "level": "log", + "source": "client", "type": "log"} + + assert.Equal(t, expectedData, data) +} + +func TestRoundTrip(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Print(">request: ", r) + if r.URL.String() == "/good" { + fmt.Fprintln(w, "Hello, client") + } + })) + defer ts.Close() + + client := http.Client{} + telemetry := NewTelemetry(EnableNetworkTelemetry(&client)) + + req := httptest.NewRequest("GET", ts.URL+"/good", nil) + res, err := telemetry.RoundTrip(req) + assert.Nil(t, err) + assert.NotNil(t, res) + + body, err := ioutil.ReadAll(res.Body) + assert.Nil(t, err) + assert.Equal(t, "Hello, client\n", string(body)) + + items := telemetry.GetQueueItems() + assert.NotNil(t, items) + + item := items[0] + expectedData := telemetry.populateTransporterBody(req, res) + assert.Equal(t, item, expectedData) +} From 3b3f4c87f6e55ffcca80b98dea2ab9f799e0fc7a Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 16 Apr 2021 17:15:34 -0700 Subject: [PATCH 08/12] cleanup timestamp --- telemetry_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/telemetry_test.go b/telemetry_test.go index 40a72ef..5b18ffd 100644 --- a/telemetry_test.go +++ b/telemetry_test.go @@ -100,7 +100,10 @@ func TestRoundTrip(t *testing.T) { items := telemetry.GetQueueItems() assert.NotNil(t, items) - item := items[0] + item := items[0].(map[string]interface{}) + delete(item, "timestamp_ms") + expectedData := telemetry.populateTransporterBody(req, res) + delete(expectedData, "timestamp_ms") assert.Equal(t, item, expectedData) } From b9aedc6d9efe14fd3d38d268dfd0808af63c6657 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 16 Apr 2021 18:52:00 -0700 Subject: [PATCH 09/12] checking scrubHeaders on telemetry too --- client_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client_test.go b/client_test.go index 7f9a770..c4889d4 100644 --- a/client_test.go +++ b/client_test.go @@ -4,12 +4,13 @@ import ( "context" "errors" "fmt" - "github.com/rollbar/rollbar-go" "net/http" "reflect" "regexp" "strings" "testing" + + "github.com/rollbar/rollbar-go" ) type TestTransport struct { @@ -435,6 +436,7 @@ func testGettersAndSetters(client *rollbar.Client, t *testing.T) { errorIfEqual(fingerprint, client.Fingerprint(), t) errorIfEqual(captureIP, client.CaptureIp(), t) errorIfEqual(scrubHeaders, client.ScrubHeaders(), t) + errorIfNotEqual(scrubHeaders, client.Telemetry.Network.ScrubHeaders, t) errorIfEqual(scrubFields, client.ScrubFields(), t) if client.Fingerprint() { @@ -476,6 +478,7 @@ func testGettersAndSetters(client *rollbar.Client, t *testing.T) { errorIfNotEqual(fingerprint, client.Fingerprint(), t) errorIfNotEqual(captureIP, client.CaptureIp(), t) errorIfNotEqual(scrubHeaders, client.ScrubHeaders(), t) + errorIfNotEqual(scrubHeaders, client.Telemetry.Network.ScrubHeaders, t) errorIfNotEqual(scrubFields, client.ScrubFields(), t) if !client.Fingerprint() { From 336c5710a2d3be85825868698efaffb8fc0a95cb Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 16 Apr 2021 22:24:16 -0700 Subject: [PATCH 10/12] corrected unit tests --- client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client_test.go b/client_test.go index c4889d4..e76a113 100644 --- a/client_test.go +++ b/client_test.go @@ -436,7 +436,7 @@ func testGettersAndSetters(client *rollbar.Client, t *testing.T) { errorIfEqual(fingerprint, client.Fingerprint(), t) errorIfEqual(captureIP, client.CaptureIp(), t) errorIfEqual(scrubHeaders, client.ScrubHeaders(), t) - errorIfNotEqual(scrubHeaders, client.Telemetry.Network.ScrubHeaders, t) + errorIfEqual(scrubHeaders, client.Telemetry.Network.ScrubHeaders, t) errorIfEqual(scrubFields, client.ScrubFields(), t) if client.Fingerprint() { From 021153045dc436b6afa4670ac99ec2b7aa141b83 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 16 Apr 2021 23:38:07 -0700 Subject: [PATCH 11/12] final tests, making it right for scrub headers --- client.go | 6 +++--- client_test.go | 18 ++++++++++++++++++ telemetry.go | 8 +++++--- telemetry_test.go | 10 +++++----- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/client.go b/client.go index b083d0d..6793c41 100644 --- a/client.go +++ b/client.go @@ -42,7 +42,7 @@ func NewAsync(token, environment, codeVersion, serverHost, serverRoot string) *C diagnostic := createDiagnostic() return &Client{ Transport: transport, - Telemetry: NewTelemetry(), + Telemetry: NewTelemetry(nil), configuration: configuration, diagnostic: diagnostic, } @@ -55,7 +55,7 @@ func NewSync(token, environment, codeVersion, serverHost, serverRoot string) *Cl diagnostic := createDiagnostic() return &Client{ Transport: transport, - Telemetry: NewTelemetry(), + Telemetry: NewTelemetry(nil), configuration: configuration, diagnostic: diagnostic, } @@ -75,7 +75,7 @@ func (c *Client) CaptureTelemetryEvent(eventType, eventlevel string, eventData m // SetTelemetry sets the telemetry func (c *Client) SetTelemetry(options ...OptionFunc) { - c.Telemetry = NewTelemetry(options...) + c.Telemetry = NewTelemetry(c.configuration.scrubHeaders, options...) } // SetEnabled sets whether or not Rollbar is enabled. diff --git a/client_test.go b/client_test.go index e76a113..e213bdd 100644 --- a/client_test.go +++ b/client_test.go @@ -465,6 +465,7 @@ func testGettersAndSetters(client *rollbar.Client, t *testing.T) { client.SetScrubHeaders(scrubHeaders) client.SetScrubFields(scrubFields) client.SetCaptureIp(captureIP) + client.SetTelemetry() client.SetEnabled(true) @@ -680,6 +681,23 @@ func TestEnabled(t *testing.T) { } } +func TestCaptureTelemetryEvent(t *testing.T) { + client := testClient() + data := map[string]interface{}{"message": "some message"} + client.CaptureTelemetryEvent("eventType", "eventLevel", data) + items := client.Telemetry.GetQueueItems() + if len(items) < 1 { + t.Error("Queue should not be empty") + } + item := items[0].(map[string]interface{}) + delete(item, "timestamp_ms") + expectedData := map[string]interface{}{"body": data, "type": "eventType", "level": "eventLevel", "source": "client"} + eq := reflect.DeepEqual(item, expectedData) + if !eq { + t.Error("Maps are different") + } +} + func configuredOptionsFromData(data map[string]interface{}) map[string]interface{} { notifier := data["notifier"].(map[string]interface{}) diagnostic := notifier["diagnostic"].(map[string]interface{}) diff --git a/telemetry.go b/telemetry.go index 7df8f64..cb589d1 100644 --- a/telemetry.go +++ b/telemetry.go @@ -152,8 +152,8 @@ func EnableLoggerTelemetry() OptionFunc { } } -// NewTelemetry initializes telemetry object -func NewTelemetry(options ...OptionFunc) *Telemetry { +// NewTelemetry initializes telemetry object with scrubheader +func NewTelemetry(scrubHeaders *regexp.Regexp, options ...OptionFunc) *Telemetry { res := &Telemetry{ Queue: NewQueue(TelemetryQueueSize), } @@ -162,8 +162,10 @@ func NewTelemetry(options ...OptionFunc) *Telemetry { opt(res) } - if res.Network.ScrubHeaders == nil { // set/define only once + if scrubHeaders == nil { res.Network.ScrubHeaders = regexp.MustCompile("Authorization") + } else { + res.Network.ScrubHeaders = scrubHeaders } return res diff --git a/telemetry_test.go b/telemetry_test.go index 5b18ffd..d56567e 100644 --- a/telemetry_test.go +++ b/telemetry_test.go @@ -14,7 +14,7 @@ import ( ) func TestNewTelemetryDefault(t *testing.T) { - telemetry := NewTelemetry() + telemetry := NewTelemetry(nil) assert.NotNil(t, telemetry) expectedTelemetry := &Telemetry{Queue: NewQueue(TelemetryQueueSize)} expectedTelemetry.Network.ScrubHeaders = regexp.MustCompile("Authorization") @@ -25,7 +25,7 @@ func TestNewTelemetryDefault(t *testing.T) { func TestNewTelemetryWithOptions(t *testing.T) { client := http.Client{} - telemetry := NewTelemetry(SetCustomQueueSize(100), EnableNetworkTelemetry(&client), + telemetry := NewTelemetry(nil, SetCustomQueueSize(100), EnableNetworkTelemetry(&client), EnableNetworkTelemetryRequestHeaders(), EnableNetworkTelemetryResponseHeaders(), EnableLoggerTelemetry()) expectedTelemetry := &Telemetry{Queue: telemetry.Queue} expectedTelemetry.Network.ScrubHeaders = regexp.MustCompile("Authorization") @@ -42,7 +42,7 @@ func TestPopulateBody(t *testing.T) { req.Header.Set("Some_name", "some_value") rec := httptest.NewRecorder() - telemetry := NewTelemetry() + telemetry := NewTelemetry(nil) EnableNetworkTelemetryRequestHeaders()(telemetry) EnableNetworkTelemetryResponseHeaders()(telemetry) data := telemetry.populateTransporterBody(req, rec.Result()) @@ -62,7 +62,7 @@ func TestPopulateBody(t *testing.T) { func TestPopulateLoggerBody(t *testing.T) { message := "some message" - telemetry := NewTelemetry() + telemetry := NewTelemetry(nil) data := telemetry.populateLoggerBody([]byte(message)) @@ -86,7 +86,7 @@ func TestRoundTrip(t *testing.T) { defer ts.Close() client := http.Client{} - telemetry := NewTelemetry(EnableNetworkTelemetry(&client)) + telemetry := NewTelemetry(nil, EnableNetworkTelemetry(&client)) req := httptest.NewRequest("GET", ts.URL+"/good", nil) res, err := telemetry.RoundTrip(req) From ffbc4114f07f1641b56ba9fda4e2c7de5082e96b Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 16 Apr 2021 23:46:35 -0700 Subject: [PATCH 12/12] added Write test --- telemetry_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/telemetry_test.go b/telemetry_test.go index d56567e..dea4cbf 100644 --- a/telemetry_test.go +++ b/telemetry_test.go @@ -107,3 +107,22 @@ func TestRoundTrip(t *testing.T) { delete(expectedData, "timestamp_ms") assert.Equal(t, item, expectedData) } + +func TestWrite(t *testing.T) { + telemetry := NewTelemetry(nil, EnableLoggerTelemetry()) + message := "some message" + count, err := telemetry.Write([]byte(message)) + + assert.Nil(t, err) + assert.Equal(t, count, len(message)) + + items := telemetry.GetQueueItems() + assert.NotNil(t, items) + + item := items[0].(map[string]interface{}) + delete(item, "timestamp_ms") + + expectedData := telemetry.populateLoggerBody([]byte(message)) + delete(expectedData, "timestamp_ms") + assert.Equal(t, item, expectedData) +}