Send alerts for failed GitHub Actions

src/api/deploy-test-suite/controllers/deploy-test-suite.js

@@ -57,9 +57,10 @@ const deployTestSuiteController = {
 
     const topic = config.get('snsRunTestTopicArn')
     const snsResponse = await sendSnsMessage({
-      request,
+      snsClient: request.snsClient,
       topic,
-      message: runMessage
+      message: runMessage,
+      logger: request.logger
     })
     request.logger.info(
       `SNS Run Test response: ${JSON.stringify(snsResponse, null, 2)}`

src/api/deploy-test-suite/controllers/stop-test-suite.js

@@ -48,9 +48,10 @@ const stopTestSuiteController = {
     request.logger.info(`Stopping task ${taskId} in ${testRun.environment}`)
 
     const snsResponse = await sendSnsMessage({
-      request,
+      snsClient: request.snsClient,
       topic,
       message,
+      logger: request.logger,
       environment: testRun.environment,
       deduplicationId: taskId
     })

src/api/deploy/helpers/send-sns-deployment-message.js

@@ -41,9 +41,10 @@ async function sendSnsDeploymentMessage(
 
   const topic = config.get('snsDeployTopicArn')
   const snsResponse = await sendSnsMessage({
-    request,
+    snsClient: request.snsClient,
     topic,
-    message: deployMessage
+    message: deployMessage,
+    logger: request.logger
   })
   const { logger } = request
 

src/api/secrets/controllers/add-secret.js

@@ -37,7 +37,7 @@ const addSecretController = {
 
     try {
       await sendSnsMessage({
-        request,
+        snsClient: request.snsClient,
         topic,
         message: {
           environment,
@@ -46,7 +46,8 @@ const addSecretController = {
           secret_key: secretKey,
           secret_value: secretValue,
           action: 'add_secret'
-        }
+        },
+        logger: request.logger
       })
 
       await registerPendingSecret({

src/config/config.js

@@ -158,6 +158,12 @@ const config = convict({
     default: 'arn:aws:sns:eu-west-2:000000000000:deploy-topic',
     env: 'SNS_DEPLOY_TOPIC_ARN'
   },
+  snsCdpNotificationArn: {
+    doc: 'SNS CDP Notification Topic ARN',
+    format: String,
+    default: 'arn:aws:sns:eu-west-2:000000000000:cdp-notification',
+    env: 'SNS_CDP_NOTIFICATION_TOPIC_ARN'
+  },
   snsRunTestTopicArn: {
     doc: 'SNS Run Test Topic ARN',
     format: String,
@@ -176,6 +182,12 @@ const config = convict({
     default: 'arn:aws:sns:eu-west-2:000000000000:secret_management',
     env: 'SNS_SECRETS_MANAGEMENT_TOPIC_ARN'
   },
+  sendFailedActionNotification: {
+    doc: 'Send notification for failed GitHub Action',
+    format: Boolean,
+    default: false,
+    env: 'SEND_FAILED_ACTION_NOTIFICATION'
+  },
   platformGlobalSecretKeys: {
     doc: 'Global Platform level secret keys. These keys are not to be overridden',
     format: Array,

src/helpers/create/workflows/trigger-workflow.js

@@ -3,9 +3,9 @@ import { octokit } from '~/src/helpers/oktokit'
 import { config } from '~/src/config'
 
 /**
- * Trigger a given github workflow
- * @param {string} org        - github org the workflow is in
- * @param {string} repo       - name of the github repo the workflow is in
+ * Trigger a given GitHub workflow
+ * @param {string} org        - GitHub org the workflow is in
+ * @param {string} repo       - name of the GitHub repo the workflow is in
  * @param {string} workflowId - name of the workflow file to trigger
  * @param {object} inputs     - input params to pass to the workflow
  */

src/helpers/sns/send-sns-message.js

@@ -2,13 +2,13 @@ import crypto from 'node:crypto'
 import { PublishCommand } from '@aws-sdk/client-sns'
 
 async function sendSnsMessage({
-  request,
+  snsClient,
   topic,
   message,
+  logger,
   environment = message?.environment,
   deduplicationId = crypto.randomUUID()
 }) {
-  const { snsClient, logger } = request
   const input = {
     TopicArn: topic,
     Message: JSON.stringify(message, null, 2),

src/listeners/github/handlers/workflow-run-alert-handler.js

@@ -0,0 +1,145 @@
+import { config } from '~/src/config'
+import { sendSnsMessage } from '~/src/helpers/sns/send-sns-message'
+
+function generateSlackMessage({
+  slackChannel,
+  workflowName,
+  repo,
+  workflowUrl,
+  runNumber,
+  commitMessage,
+  author
+}) {
+  return {
+    channel: slackChannel,
+    attachments: [
+      {
+        color: '#f03f36',
+        blocks: [
+          {
+            type: 'context',
+            elements: [
+              {
+                type: 'image',
+                image_url:
+                  'https://www.iconfinder.com/icons/298822/download/png/512',
+                alt_text: 'GitHub'
+              },
+              {
+                type: 'mrkdwn',
+                text: '*Failed GitHub Action*'
+              }
+            ]
+          },
+          {
+            type: 'rich_text',
+            elements: [
+              {
+                type: 'rich_text_section',
+                elements: [
+                  {
+                    type: 'text',
+                    text: `${repo} - '${workflowName}' failed`,
+                    style: {
+                      bold: true
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            type: 'context',
+            elements: [
+              {
+                type: 'mrkdwn',
+                text: `*Failed Workflow:* <${workflowUrl}|${runNumber}>`
+              }
+            ]
+          },
+          {
+            type: 'context',
+            elements: [
+              {
+                type: 'mrkdwn',
+                text: `*Commit Message:* '${commitMessage}'\n*Author:* ${author}`
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}
+
+const failedConclusions = new Set([
+  'action_required',
+  'failure',
+  'stale',
+  'timed_out',
+  'startup_failure'
+])
+
+function shouldSendAlert(headBranch, action, conclusion) {
+  return (
+    headBranch === 'main' &&
+    action === 'completed' &&
+    failedConclusions.has(conclusion)
+  )
+}
+
+async function workflowRunAlertHandler(server, event) {
+  const { name: repo, html_url: repoUrl } = event.repository
+  const action = event.action
+  const {
+    head_branch: headBranch,
+    name: workflowName,
+    html_url: workflowUrl,
+    run_number: runNumber,
+    head_commit: headCommit,
+    conclusion
+  } = event.workflow_run
+
+  const commitMessage = headCommit.message
+  const author = headCommit.author.name
+
+  const slackChannel = 'cdp-platform-alerts'
+
+  const sendFailedActionNotification = config.get(
+    'sendFailedActionNotification'
+  )
+  const ActionFailed = shouldSendAlert(headBranch, action, conclusion)
+
+  if (ActionFailed) {
+    server.logger.info(
+      `Workflow '${workflowName}' in ${repo} failed: ${workflowUrl}`
+    )
+  }
+
+  if (ActionFailed && sendFailedActionNotification) {
+    const message = generateSlackMessage({
+      slackChannel,
+      workflowName,
+      repo,
+      repoUrl,
+      workflowUrl,
+      runNumber,
+      commitMessage,
+      author
+    })
+
+    const topic = config.get('snsCdpNotificationArn')
+    await sendSnsMessage({
+      snsClient: server.snsClient,
+      topic,
+      message: {
+        team: 'platform',
+        slack_channel: slackChannel,
+        message
+      },
+      logger: server.logger
+    })
+  }
+}
+
+export { workflowRunAlertHandler }

src/listeners/github/message-handler.js

@@ -1,5 +1,6 @@
 import { config } from '~/src/config'
 import { workflowRunHandlerV2 } from '~/src/listeners/github/handlers/workflow-run-handler-v2'
+import { workflowRunAlertHandler } from '~/src/listeners/github/handlers/workflow-run-alert-handler'
 
 const githubWebhooks = new Set([
   config.get('github.repos.appDeployments'),
@@ -10,22 +11,19 @@ const githubWebhooks = new Set([
   config.get('github.repos.cdpSquidProxy'),
   config.get('github.repos.cdpGrafanaSvc')
 ])
-const validActions = new Set(['workflow_run'])
 
-const shouldProcess = (message) => {
-  const eventType = message.github_event
+const shouldProcess = (message, eventType) => {
   const repo = message.repository?.name
-  return validActions.has(eventType) && githubWebhooks.has(repo)
+  return message.github_event === eventType && githubWebhooks.has(repo)
 }
 
 const handle = async (server, message) => {
-  if (!shouldProcess(message)) {
-    return
-  }
-
-  if (message.github_event === 'workflow_run') {
+  if (shouldProcess(message, 'workflow_run')) {
     return await workflowRunHandlerV2(server, message)
   }
+  if (shouldProcess(message, 'workflow_run')) {
+    return await workflowRunAlertHandler(server, message)
+  }
 }
 
 export { handle }

src/plugins/sns-client.js

@@ -16,6 +16,7 @@ const snsClientPlugin = {
 
       server.logger.info('sns-client configured')
       server.decorate('request', 'snsClient', client)
+      server.decorate('server', 'snsClient', client)
 
       server.events.on('stop', () => {
         server.logger.info(`Closing SNS client`)