diff --git a/src/api/deploy-test-suite/controllers/deploy-test-suite.js b/src/api/deploy-test-suite/controllers/deploy-test-suite.js index 237f053..44b5b89 100644 --- a/src/api/deploy-test-suite/controllers/deploy-test-suite.js +++ b/src/api/deploy-test-suite/controllers/deploy-test-suite.js @@ -57,9 +57,10 @@ const deployTestSuiteController = { const topic = config.get('snsRunTestTopicArn') const snsResponse = await sendSnsMessage({ - request, + snsClient: request.snsClient, topic, - message: runMessage + message: runMessage, + logger: request.logger }) request.logger.info( `SNS Run Test response: ${JSON.stringify(snsResponse, null, 2)}` diff --git a/src/api/deploy-test-suite/controllers/stop-test-suite.js b/src/api/deploy-test-suite/controllers/stop-test-suite.js index 9f31080..3c5f222 100644 --- a/src/api/deploy-test-suite/controllers/stop-test-suite.js +++ b/src/api/deploy-test-suite/controllers/stop-test-suite.js @@ -48,9 +48,10 @@ const stopTestSuiteController = { request.logger.info(`Stopping task ${taskId} in ${testRun.environment}`) const snsResponse = await sendSnsMessage({ - request, + snsClient: request.snsClient, topic, message, + logger: request.logger, environment: testRun.environment, deduplicationId: taskId }) diff --git a/src/api/deploy/helpers/send-sns-deployment-message.js b/src/api/deploy/helpers/send-sns-deployment-message.js index 80d5ec1..ee3bfba 100644 --- a/src/api/deploy/helpers/send-sns-deployment-message.js +++ b/src/api/deploy/helpers/send-sns-deployment-message.js @@ -41,9 +41,10 @@ async function sendSnsDeploymentMessage( const topic = config.get('snsDeployTopicArn') const snsResponse = await sendSnsMessage({ - request, + snsClient: request.snsClient, topic, - message: deployMessage + message: deployMessage, + logger: request.logger }) const { logger } = request diff --git a/src/api/secrets/controllers/add-secret.js b/src/api/secrets/controllers/add-secret.js index adf7b1f..9a1a705 100644 --- a/src/api/secrets/controllers/add-secret.js +++ b/src/api/secrets/controllers/add-secret.js @@ -37,7 +37,7 @@ const addSecretController = { try { await sendSnsMessage({ - request, + snsClient: request.snsClient, topic, message: { environment, @@ -46,7 +46,8 @@ const addSecretController = { secret_key: secretKey, secret_value: secretValue, action: 'add_secret' - } + }, + logger: request.logger }) await registerPendingSecret({ diff --git a/src/config/config.js b/src/config/config.js index 8cdd172..90f0194 100644 --- a/src/config/config.js +++ b/src/config/config.js @@ -158,6 +158,12 @@ const config = convict({ default: 'arn:aws:sns:eu-west-2:000000000000:deploy-topic', env: 'SNS_DEPLOY_TOPIC_ARN' }, + snsCdpNotificationArn: { + doc: 'SNS CDP Notification Topic ARN', + format: String, + default: 'arn:aws:sns:eu-west-2:000000000000:cdp-notification', + env: 'SNS_CDP_NOTIFICATION_TOPIC_ARN' + }, snsRunTestTopicArn: { doc: 'SNS Run Test Topic ARN', format: String, @@ -176,6 +182,12 @@ const config = convict({ default: 'arn:aws:sns:eu-west-2:000000000000:secret_management', env: 'SNS_SECRETS_MANAGEMENT_TOPIC_ARN' }, + sendFailedActionNotification: { + doc: 'Send notification for failed GitHub Action', + format: Boolean, + default: false, + env: 'SEND_FAILED_ACTION_NOTIFICATION' + }, platformGlobalSecretKeys: { doc: 'Global Platform level secret keys. These keys are not to be overridden', format: Array, diff --git a/src/helpers/create/workflows/trigger-workflow.js b/src/helpers/create/workflows/trigger-workflow.js index cff9ec1..13f8593 100644 --- a/src/helpers/create/workflows/trigger-workflow.js +++ b/src/helpers/create/workflows/trigger-workflow.js @@ -3,9 +3,9 @@ import { octokit } from '~/src/helpers/oktokit' import { config } from '~/src/config' /** - * Trigger a given github workflow - * @param {string} org - github org the workflow is in - * @param {string} repo - name of the github repo the workflow is in + * Trigger a given GitHub workflow + * @param {string} org - GitHub org the workflow is in + * @param {string} repo - name of the GitHub repo the workflow is in * @param {string} workflowId - name of the workflow file to trigger * @param {object} inputs - input params to pass to the workflow */ diff --git a/src/helpers/sns/send-sns-message.js b/src/helpers/sns/send-sns-message.js index 9fbb78c..7530c07 100644 --- a/src/helpers/sns/send-sns-message.js +++ b/src/helpers/sns/send-sns-message.js @@ -2,13 +2,13 @@ import crypto from 'node:crypto' import { PublishCommand } from '@aws-sdk/client-sns' async function sendSnsMessage({ - request, + snsClient, topic, message, + logger, environment = message?.environment, deduplicationId = crypto.randomUUID() }) { - const { snsClient, logger } = request const input = { TopicArn: topic, Message: JSON.stringify(message, null, 2), diff --git a/src/listeners/github/handlers/workflow-run-alert-handler.js b/src/listeners/github/handlers/workflow-run-alert-handler.js new file mode 100644 index 0000000..4812470 --- /dev/null +++ b/src/listeners/github/handlers/workflow-run-alert-handler.js @@ -0,0 +1,145 @@ +import { config } from '~/src/config' +import { sendSnsMessage } from '~/src/helpers/sns/send-sns-message' + +function generateSlackMessage({ + slackChannel, + workflowName, + repo, + workflowUrl, + runNumber, + commitMessage, + author +}) { + return { + channel: slackChannel, + attachments: [ + { + color: '#f03f36', + blocks: [ + { + type: 'context', + elements: [ + { + type: 'image', + image_url: + 'https://www.iconfinder.com/icons/298822/download/png/512', + alt_text: 'GitHub' + }, + { + type: 'mrkdwn', + text: '*Failed GitHub Action*' + } + ] + }, + { + type: 'rich_text', + elements: [ + { + type: 'rich_text_section', + elements: [ + { + type: 'text', + text: `${repo} - '${workflowName}' failed`, + style: { + bold: true + } + } + ] + } + ] + }, + { + type: 'context', + elements: [ + { + type: 'mrkdwn', + text: `*Failed Workflow:* <${workflowUrl}|${runNumber}>` + } + ] + }, + { + type: 'context', + elements: [ + { + type: 'mrkdwn', + text: `*Commit Message:* '${commitMessage}'\n*Author:* ${author}` + } + ] + } + ] + } + ] + } +} + +const failedConclusions = new Set([ + 'action_required', + 'failure', + 'stale', + 'timed_out', + 'startup_failure' +]) + +function shouldSendAlert(headBranch, action, conclusion) { + return ( + headBranch === 'main' && + action === 'completed' && + failedConclusions.has(conclusion) + ) +} + +async function workflowRunAlertHandler(server, event) { + const { name: repo, html_url: repoUrl } = event.repository + const action = event.action + const { + head_branch: headBranch, + name: workflowName, + html_url: workflowUrl, + run_number: runNumber, + head_commit: headCommit, + conclusion + } = event.workflow_run + + const commitMessage = headCommit.message + const author = headCommit.author.name + + const slackChannel = 'cdp-platform-alerts' + + const sendFailedActionNotification = config.get( + 'sendFailedActionNotification' + ) + const ActionFailed = shouldSendAlert(headBranch, action, conclusion) + + if (ActionFailed) { + server.logger.info( + `Workflow '${workflowName}' in ${repo} failed: ${workflowUrl}` + ) + } + + if (ActionFailed && sendFailedActionNotification) { + const message = generateSlackMessage({ + slackChannel, + workflowName, + repo, + repoUrl, + workflowUrl, + runNumber, + commitMessage, + author + }) + + const topic = config.get('snsCdpNotificationArn') + await sendSnsMessage({ + snsClient: server.snsClient, + topic, + message: { + team: 'platform', + slack_channel: slackChannel, + message + }, + logger: server.logger + }) + } +} + +export { workflowRunAlertHandler } diff --git a/src/listeners/github/message-handler.js b/src/listeners/github/message-handler.js index cfae792..ced64a8 100644 --- a/src/listeners/github/message-handler.js +++ b/src/listeners/github/message-handler.js @@ -1,5 +1,6 @@ import { config } from '~/src/config' import { workflowRunHandlerV2 } from '~/src/listeners/github/handlers/workflow-run-handler-v2' +import { workflowRunAlertHandler } from '~/src/listeners/github/handlers/workflow-run-alert-handler' const githubWebhooks = new Set([ config.get('github.repos.appDeployments'), @@ -10,21 +11,22 @@ const githubWebhooks = new Set([ config.get('github.repos.cdpSquidProxy'), config.get('github.repos.cdpGrafanaSvc') ]) -const validActions = new Set(['workflow_run']) -const shouldProcess = (message) => { - const eventType = message.github_event +const shouldProcess = (message, eventType) => { const repo = message.repository?.name - return validActions.has(eventType) && githubWebhooks.has(repo) + return message.github_event === eventType && githubWebhooks.has(repo) } const handle = async (server, message) => { - if (!shouldProcess(message)) { - return - } - - if (message.github_event === 'workflow_run') { - return await workflowRunHandlerV2(server, message) + try { + if (shouldProcess(message, 'workflow_run')) { + return await workflowRunHandlerV2(server, message) + } + if (shouldProcess(message, 'workflow_run')) { + return await workflowRunAlertHandler(server, message) + } + } catch (error) { + server.logger.error(error, 'Exception in workflow handler') } } diff --git a/src/plugins/sns-client.js b/src/plugins/sns-client.js index 74f0733..c5a05ca 100644 --- a/src/plugins/sns-client.js +++ b/src/plugins/sns-client.js @@ -16,6 +16,7 @@ const snsClientPlugin = { server.logger.info('sns-client configured') server.decorate('request', 'snsClient', client) + server.decorate('server', 'snsClient', client) server.events.on('stop', () => { server.logger.info(`Closing SNS client`)