From 28802b7f23dde8f4aeebc6aa2e88686872b59b25 Mon Sep 17 00:00:00 2001 From: Nick Vrvilo Date: Mon, 28 Nov 2022 13:30:30 -0600 Subject: [PATCH] logs and metrics for k8s api 5XX responses (#1687) --- waiter/src/waiter/scheduler/kubernetes.clj | 6 ++++++ waiter/src/waiter/util/http_utils.clj | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/waiter/src/waiter/scheduler/kubernetes.clj b/waiter/src/waiter/scheduler/kubernetes.clj index 8c5a6fb49..6e6563f7f 100644 --- a/waiter/src/waiter/scheduler/kubernetes.clj +++ b/waiter/src/waiter/scheduler/kubernetes.clj @@ -519,6 +519,9 @@ {:keys [body error status] :as response} (clj-http/get url request-options)] (when error (throw error)) + (when (hu/status-5XX? status) + (log/warn "received 5XX response from k8s api server streaming request" {:url url :response response}) + (counters/inc! (metrics/waiter-counter "k8s-api" "5XX"))) (when-not (hu/status-2XX? status) (ss/throw+ response)) (-> body @@ -577,6 +580,9 @@ (ss/throw+ response)) (catch [:client http-client] response (log/error "request to K8s API server failed: " url options body response) + (when (hu/status-5XX? (:status response)) + (log/warn "received 5XX response from k8s api server request" {:url url :response response}) + (counters/inc! (metrics/waiter-counter "k8s-api" "5XX"))) (ss/throw+ response)))) (defn api-request-async diff --git a/waiter/src/waiter/util/http_utils.clj b/waiter/src/waiter/util/http_utils.clj index 72e13986b..ac0fc394b 100644 --- a/waiter/src/waiter/util/http_utils.clj +++ b/waiter/src/waiter/util/http_utils.clj @@ -73,6 +73,12 @@ (and (integer? status) (<= http-200-ok status 299))) +(defn status-5XX? + "Returns true if the status is in the range [500, 599]." + [status] + (and (integer? status) + (<= http-500-internal-server-error status 599))) + (defn http-streaming-request-async "This wraps the qbits.jet.client.http/request function and makes a streaming request. An abort-ch can be provided to abort the streaming response when a message is pushed to it.