From 489db64b8f409abd58aa74712a78cb7e7178d76b Mon Sep 17 00:00:00 2001
From: Pat Heard <patrick.heard@cds-snc.ca>
Date: Thu, 13 Jun 2024 14:57:17 -0400
Subject: [PATCH] feat: add CloudWatch metrics for Lambda behaviour (#683)

Add CloudWatch metric filters for the following Lambda functions:

- Form archiver
- Reliability
- Response archiver
- Submission

These will be used to build out dashboards and create health check
alarms if success metrics are not observed consistency.

Add success log events and a `status` property to the form-archiver
and response-archiver Lambda functions.
---
 aws/alarms/metrics.tf                 | 73 ++++++++++++++++++++++++++-
 lambda-code/form-archiver/main.ts     |  9 ++++
 lambda-code/response-archiver/main.ts |  9 ++++
 3 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/aws/alarms/metrics.tf b/aws/alarms/metrics.tf
index bb3a45a54..949fb6de2 100644
--- a/aws/alarms/metrics.tf
+++ b/aws/alarms/metrics.tf
@@ -3,13 +3,82 @@
 #
 locals {
   healthcheck_metrics = [{
-    name           = "FormsClientSubmitSuccess"
+    # ECS: client
+    name           = "ClientSubmitSuccess"
     pattern        = "Response submitted for Form ID"
     log_group_name = var.ecs_cloudwatch_log_group_name
     }, {
-    name           = "FormsClientSubmitFailed"
+    name           = "ClientSubmitFailed"
     pattern        = "Attempted response submission for Form ID"
     log_group_name = var.ecs_cloudwatch_log_group_name
+    }, {
+    # Lambda: form-archiver
+    name           = "FormArchiverSuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_form_archiver_log_group_name
+    }, {
+    name           = "FormArchiverWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_form_archiver_log_group_name
+    }, {
+    name           = "FormArchiverFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_form_archiver_log_group_name
+    }, {
+    # Lambda: reliability
+    name           = "ReliabilitySuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityNotifySendSuccess"
+    pattern        = "Successfully sent submission through GC Notify"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityNotifySendFailed"
+    pattern        = "Failed to send submission through GC Notify"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityVaultSaveSuccess"
+    pattern        = "Successfully saved submission to Vault"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    name           = "ReliabilityVaultSaveFailed"
+    pattern        = "Failed to save submission to Vault"
+    log_group_name = var.lambda_reliability_log_group_name
+    }, {
+    # Lambda: response-archiver
+    name           = "ResponseArchiverSuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_response_archiver_log_group_name
+    }, {
+    name           = "ResponseArchiverWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_response_archiver_log_group_name
+    }, {
+    name           = "ResponseArchiverFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_response_archiver_log_group_name
+    }, {
+    # Lambda: submission
+    name           = "SubmissionSuccess"
+    pattern        = "{$.status = \"success\"}"
+    log_group_name = var.lambda_submission_log_group_name
+    }, {
+    name           = "SubmissionWarn"
+    pattern        = "{$.level = \"warn\"}"
+    log_group_name = var.lambda_submission_log_group_name
+    }, {
+    name           = "SubmissionFailed"
+    pattern        = "{$.status = \"failed\"}"
+    log_group_name = var.lambda_submission_log_group_name
   }]
 }
 
diff --git a/lambda-code/form-archiver/main.ts b/lambda-code/form-archiver/main.ts
index d502f9cef..a07a68f73 100644
--- a/lambda-code/form-archiver/main.ts
+++ b/lambda-code/form-archiver/main.ts
@@ -5,6 +5,14 @@ export const handler: Handler = async () => {
   try {
     await deleteFormTemplatesMarkedAsArchived();
 
+    console.log(
+      JSON.stringify({
+        level: "info",
+        status: "success",
+        msg: "Form Archiver ran successfully.",
+      })
+    );
+
     return {
       statusCode: "SUCCESS",
     };
@@ -13,6 +21,7 @@ export const handler: Handler = async () => {
     console.error(
       JSON.stringify({
         level: "error",
+        status: "failed",
         msg: "Failed to run Form Templates Archiver.",
         error: (error as Error).message,
       })
diff --git a/lambda-code/response-archiver/main.ts b/lambda-code/response-archiver/main.ts
index 48f505358..690407598 100644
--- a/lambda-code/response-archiver/main.ts
+++ b/lambda-code/response-archiver/main.ts
@@ -43,6 +43,14 @@ export const handler: Handler = async () => {
 
     await archiveResponses(dynamodbClient, s3Client);
 
+    console.log(
+      JSON.stringify({
+        level: "info",
+        status: "success",
+        msg: "Response Archiver ran successfully.",
+      })
+    );
+
     return {
       statusCode: "SUCCESS",
     };
@@ -51,6 +59,7 @@ export const handler: Handler = async () => {
     console.error(
       JSON.stringify({
         level: "error",
+        status: "failed",
         msg: "Failed to run Form Responses Archiver.",
         error: (error as Error).message,
       })