feat: add CloudWatch metrics for Lambda behaviour (#683)

Add CloudWatch metric filters for the following Lambda functions: - Form archiver - Reliability - Response archiver - Submission These will be used to build out dashboards and create health check alarms if success metrics are not observed consistency. Add success log events and a `status` property to the form-archiver and response-archiver Lambda functions.
cds-snc · Jun 13, 2024 · 489db64 · 489db64
1 parent bde87ea
commit 489db64
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 2 deletions.
diff --git a/aws/alarms/metrics.tf b/aws/alarms/metrics.tf
@@ -3,13 +3,82 @@
 #
 locals {
   healthcheck_metrics = [{
-    name           = "FormsClientSubmitSuccess"
+    # ECS: client
+    name           = "ClientSubmitSuccess"
     pattern        = "Response submitted for Form ID"
     log_group_name = var.ecs_cloudwatch_log_group_name
     }, {
-    name           = "FormsClientSubmitFailed"
+    name           = "ClientSubmitFailed"
     pattern        = "Attempted response submission for Form ID"
     log_group_name = var.ecs_cloudwatch_log_group_name
+    }, {
+    # Lambda: form-archiver
+    name           = "FormArchiverSuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_form_archiver_log_group_name
+    }, {
+    name           = "FormArchiverWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_form_archiver_log_group_name
+    }, {
+    name           = "FormArchiverFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_form_archiver_log_group_name
+    }, {
+    # Lambda: reliability
+    name           = "ReliabilitySuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityNotifySendSuccess"
+    pattern        = "Successfully sent submission through GC Notify"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityNotifySendFailed"
+    pattern        = "Failed to send submission through GC Notify"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityVaultSaveSuccess"
+    pattern        = "Successfully saved submission to Vault"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityVaultSaveFailed"
+    pattern        = "Failed to save submission to Vault"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    # Lambda: response-archiver
+    name           = "ResponseArchiverSuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_response_archiver_log_group_name
+    }, {
+    name           = "ResponseArchiverWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_response_archiver_log_group_name
+    }, {
+    name           = "ResponseArchiverFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_response_archiver_log_group_name
+    }, {
+    # Lambda: submission
+    name           = "SubmissionSuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_submission_log_group_name
+    }, {
+    name           = "SubmissionWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_submission_log_group_name
+    }, {
+    name           = "SubmissionFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_submission_log_group_name
   }]
 }
 

diff --git a/lambda-code/form-archiver/main.ts b/lambda-code/form-archiver/main.ts
@@ -5,6 +5,14 @@ export const handler: Handler = async () => {
   try {
     await deleteFormTemplatesMarkedAsArchived();
 
+    console.log(
+      JSON.stringify({
+        level: "info",
+        status: "success",
+        msg: "Form Archiver ran successfully.",
+      })
+    );
+
     return {
       statusCode: "SUCCESS",
     };
@@ -13,6 +21,7 @@ export const handler: Handler = async () => {
     console.error(
       JSON.stringify({
         level: "error",
+        status: "failed",
         msg: "Failed to run Form Templates Archiver.",
         error: (error as Error).message,
       })

diff --git a/lambda-code/response-archiver/main.ts b/lambda-code/response-archiver/main.ts
@@ -43,6 +43,14 @@ export const handler: Handler = async () => {
 
     await archiveResponses(dynamodbClient, s3Client);
 
+    console.log(
+      JSON.stringify({
+        level: "info",
+        status: "success",
+        msg: "Response Archiver ran successfully.",
+      })
+    );
+
     return {
       statusCode: "SUCCESS",
     };
@@ -51,6 +59,7 @@ export const handler: Handler = async () => {
     console.error(
       JSON.stringify({
         level: "error",
+        status: "failed",
         msg: "Failed to run Form Responses Archiver.",
         error: (error as Error).message,
       })