Merge branch 'main' into ARC-2444

src/config/feature-flags.ts

@@ -37,7 +37,8 @@ export enum BooleanFlags {
 export enum StringFlags {
 	BLOCKED_INSTALLATIONS = "blocked-installations",
 	LOG_LEVEL = "log-level",
-	HEADERS_TO_ENCRYPT = "headers-to-encrypt"
+	HEADERS_TO_ENCRYPT = "headers-to-encrypt",
+	SEND_ALL = "send-all"
 }
 
 export enum NumberFlags {
@@ -90,6 +91,23 @@ export const onFlagChange = (flag: BooleanFlags | StringFlags | NumberFlags, lis
 	launchdarklyClient.on(`update:${flag}`, listener);
 };
 
+type ShouldSendAllStringTypes =
+	"branches-backfill" | "builds-backfill" | "commits-backfill" | "deployments-backfill" | "prs-backfill" |
+	"branches" | "builds" | "commits" | "deployments" | "prs";
+
+export const shouldSendAll = async (type: ShouldSendAllStringTypes, jiraHost: string, logger: Logger): Promise<boolean> => {
+	try {
+		// Full set:
+		// ["branches-backfill", "builds-backfill", "commits-backfill", "deployments-backfill", "prs-backfill", "branches", "builds", "commits", "deployments", "prs"]
+		const sendAllString: string = await stringFlag(StringFlags.SEND_ALL, "[]", jiraHost);
+		const sendAllArray: string[] = JSON.parse(sendAllString);
+		return sendAllArray.includes(type);
+	} catch (e) {
+		logger.error({ err: e, type }, "Cannot define if should send all");
+		return false;
+	}
+};
+
 export const isBlocked = async (jiraHost: string, installationId: number, logger: Logger): Promise<boolean> => {
 	try {
 		const blockedInstallationsString = await stringFlag(StringFlags.BLOCKED_INSTALLATIONS, "[]", jiraHost);

src/github/branch.ts

@@ -10,6 +10,7 @@ import { JiraBranchBulkSubmitData } from "interfaces/jira";
 import { jiraIssueKeyParser } from "utils/jira-utils";
 import { WebhookContext } from "routes/github/webhook/webhook-context";
 import { transformRepositoryId } from "~/src/transforms/transform-repository-id";
+import { shouldSendAll } from "config/feature-flags";
 
 export const createBranchWebhookHandler = async (context: WebhookContext<CreateEvent>, jiraClient, _util, gitHubInstallationId: number): Promise<void> => {
 
@@ -42,8 +43,8 @@ export const processBranch = async (
 		webhookReceived: webhookReceivedDate
 	});
 
-	const jiraPayload: JiraBranchBulkSubmitData | undefined = await transformBranch(github, webhookPayload, logger);
-
+	const alwaysSend = await shouldSendAll("branches", jiraHost, logger);
+	const jiraPayload: JiraBranchBulkSubmitData | undefined = await transformBranch(github, webhookPayload, alwaysSend, logger);
 
 	if (!jiraPayload) {
 		logger.info("Halting further execution for createBranch since jiraPayload is empty");

src/github/push.test.ts

@@ -31,7 +31,7 @@ describe("PushWebhookHandler", ()=>{
 	describe("GitHub Cloud", ()=>{
 		it("should be called with cloud GitHubAppConfig", async ()=>{
 			await pushWebhookHandler(getWebhookContext({ cloud: true }), jiraClient, util, GHES_GITHUB_INSTALLATION_ID, subscription);
-			expect(enqueuePush).toBeCalledWith(expect.anything(), expect.anything(), {
+			expect(enqueuePush).toBeCalledWith(expect.anything(), expect.anything(), expect.anything(), {
 				uuid: undefined,
 				gitHubAppId: undefined,
 				appId: parseInt(envVars.APP_ID),
@@ -44,7 +44,7 @@ describe("PushWebhookHandler", ()=>{
 	describe("GitHub Enterprise Server", ()=>{
 		it("should be called with GHES GitHubAppConfig", async ()=>{
 			await pushWebhookHandler(getWebhookContext({ cloud: false }), jiraClient, util, GHES_GITHUB_INSTALLATION_ID, subscription);
-			expect(enqueuePush).toBeCalledWith(expect.anything(), expect.anything(), {
+			expect(enqueuePush).toBeCalledWith(expect.anything(), expect.anything(), expect.anything(), {
 				uuid: GHES_GITHUB_UUID,
 				gitHubAppId: GHES_GITHUB_APP_ID,
 				appId: GHES_GITHUB_APP_APP_ID,

src/github/push.ts

@@ -68,5 +68,5 @@ export const pushWebhookHandler = async (context: WebhookContext, jiraClient, _u
 	}
 
 	context.log.info("Enqueueing push event");
-	await enqueuePush(payload, jiraClient.baseURL, context.gitHubAppConfig);
+	await enqueuePush(payload, jiraClient.baseURL, context.log, context.gitHubAppConfig);
 };

src/github/workflow.ts

@@ -2,12 +2,13 @@ import { transformWorkflow } from "../transforms/transform-workflow";
 import { emitWebhookProcessedMetrics } from "utils/webhook-utils";
 import { createInstallationClient } from "utils/get-github-client-config";
 import { WebhookContext } from "../routes/github/webhook/webhook-context";
+import { shouldSendAll } from "config/feature-flags";
 
 export const workflowWebhookHandler = async (context: WebhookContext, jiraClient, _util, gitHubInstallationId: number): Promise<void> => {
 	const { payload, log: logger } = context;
-
+	const jiraHost = jiraClient.baseURL;
 	context.log = context.log.child({
-		jiraHost: jiraClient.baseURL,
+		jiraHost,
 		gitHubInstallationId
 	});
 
@@ -17,7 +18,8 @@ export const workflowWebhookHandler = async (context: WebhookContext, jiraClient
 		subTrigger: "workflow"
 	};
 	const gitHubInstallationClient = await createInstallationClient(gitHubInstallationId, jiraClient.baseURL, metrics, context.log, gitHubAppId);
-	const jiraPayload = await transformWorkflow(gitHubInstallationClient, payload, logger);
+	const alwaysSend = await shouldSendAll("builds", jiraHost, logger);
+	const jiraPayload = await transformWorkflow(gitHubInstallationClient, payload, alwaysSend, logger);
 
 	if (!jiraPayload) {
 		logger.info(

src/interfaces/jira.ts

@@ -148,7 +148,7 @@ export interface JiraDeployment {
 		displayName: string;
 		type: string;
 	};
-	associations: JiraAssociation[];
+	associations: JiraAssociation[] | undefined;
 }
 
 export interface JiraDeploymentBulkSubmitData {

src/sqs/push.test.ts

@@ -1,4 +1,4 @@
-import nock, { removeInterceptor, cleanAll } from "nock";
+import nock, { cleanAll, removeInterceptor } from "nock";
 import { createJobData } from "../transforms/push";
 import { getLogger } from "config/logger";
 import { waitUntil } from "test/utils/wait-until";
@@ -91,10 +91,10 @@ const createJiraPayloadNoUsername = (transofmedRepoId: string) => {
 describe("Push Webhook", () => {
 
 	describe("cloud",  () => {
-
-		const createMessageProcessingContext = (payload: any): SQSMessageContext<PushQueueMessagePayload> => ({
-			payload: createJobData(updateInstallationId(payload), jiraHost),
-			log: getLogger("test"),
+		const logger = getLogger("test");
+		const createMessageProcessingContext = async (payload: any): Promise<SQSMessageContext<PushQueueMessagePayload>> => ({
+			payload: await createJobData(updateInstallationId(payload), jiraHost, logger),
+			log: logger,
 			message: {} as Message,
 			receiveCount: 1,
 			lastAttempt: false
@@ -119,7 +119,7 @@ describe("Push Webhook", () => {
 
 				jiraNock.post("/rest/devinfo/0.10/bulk", createJiraPayloadNoUsername("test-repo-id")).reply(200);
 
-				await expect(pushQueueMessageHandler(createMessageProcessingContext(pushNoUsername.payload))).toResolve();
+				await expect(pushQueueMessageHandler(await createMessageProcessingContext(pushNoUsername.payload))).toResolve();
 			});
 
 			it("should only send 10 files if push contains more than 10 files changed", async () => {
@@ -231,7 +231,7 @@ describe("Push Webhook", () => {
 					}
 				}).reply(200);
 
-				await expect(pushQueueMessageHandler(createMessageProcessingContext(pushMultiple.payload))).toResolve();
+				await expect(pushQueueMessageHandler(await createMessageProcessingContext(pushMultiple.payload))).toResolve();
 			});
 
 			it("should only files with valid file paths (not empty or undefined)", async () => {
@@ -287,7 +287,7 @@ describe("Push Webhook", () => {
 					}
 				}).reply(200);
 
-				await expect(pushQueueMessageHandler(createMessageProcessingContext(pushMultiple.payload))).toResolve();
+				await expect(pushQueueMessageHandler(await createMessageProcessingContext(pushMultiple.payload))).toResolve();
 			});
 
 			it("should truncate long file paths to 1024", async () => {
@@ -343,7 +343,7 @@ describe("Push Webhook", () => {
 					}
 				}).reply(200);
 
-				await expect(pushQueueMessageHandler(createMessageProcessingContext(pushMultiple.payload))).toResolve();
+				await expect(pushQueueMessageHandler(await createMessageProcessingContext(pushMultiple.payload))).toResolve();
 			});
 
 			it("should not run a command without a Jira issue", async () => {
@@ -427,7 +427,7 @@ describe("Push Webhook", () => {
 					properties: { installationId: DatabaseStateCreator.GITHUB_INSTALLATION_ID }
 				}).reply(200);
 
-				await expect(pushQueueMessageHandler(createMessageProcessingContext(pushNoUsername.payload))).toResolve();
+				await expect(pushQueueMessageHandler(await createMessageProcessingContext(pushNoUsername.payload))).toResolve();
 			});
 
 			it("should not add the MERGE_COMMIT flag when a commit is not a merge commit", async () => {
@@ -490,7 +490,7 @@ describe("Push Webhook", () => {
 					properties: { installationId: DatabaseStateCreator.GITHUB_INSTALLATION_ID }
 				}).reply(200);
 
-				await expect(pushQueueMessageHandler(createMessageProcessingContext(pushNoUsername.payload))).toResolve();
+				await expect(pushQueueMessageHandler(await createMessageProcessingContext(pushNoUsername.payload))).toResolve();
 			});
 		});
 
@@ -584,8 +584,8 @@ describe("Push Webhook", () => {
 
 		let gitHubServerApp: GitHubServerApp;
 
-		const createMessageProcessingContext = (payload): SQSMessageContext<PushQueueMessagePayload> => ({
-			payload: createJobData(updateInstallationId(payload), jiraHost, {
+		const createMessageProcessingContext = async (payload): Promise<SQSMessageContext<PushQueueMessagePayload>> => ({
+			payload: await createJobData(updateInstallationId(payload), jiraHost, getLogger("test"), {
 				gitHubAppId: gitHubServerApp.id,
 				appId: gitHubServerApp.appId,
 				clientId: gitHubServerApp.gitHubClientId,
@@ -620,7 +620,7 @@ describe("Push Webhook", () => {
 
 				jiraNock.post("/rest/devinfo/0.10/bulk", createJiraPayloadNoUsername("6769746875626d79646f6d61696e636f6d-test-repo-id")).reply(200);
 
-				await expect(pushQueueMessageHandler(createMessageProcessingContext(pushNoUsername.payload))).toResolve();
+				await expect(pushQueueMessageHandler(await createMessageProcessingContext(pushNoUsername.payload))).toResolve();
 			});
 
 		});

src/sync/branches.ts

@@ -4,6 +4,7 @@ import { GitHubInstallationClient } from "../github/client/github-installation-c
 import Logger from "bunyan";
 import { BackfillMessagePayload } from "~/src/sqs/sqs.types";
 import { createHashWithSharedSecret } from "utils/encryption";
+import { shouldSendAll } from "config/feature-flags";
 
 // TODO: better typings
 export const getBranchTask = async (
@@ -26,8 +27,9 @@ export const getBranchTask = async (
 	const branches = edges.map(edge => edge?.node);
 	(logger.fields || {}).branchNameArray = (branches || []).map(b => createHashWithSharedSecret(String(b.name)));
 	(logger.fields || {}).branchShaArray = (branches || []).map(b => createHashWithSharedSecret(String(b.target?.oid)));
-
-	const jiraPayload = transformBranches({ branches, repository }, messagePayload.gitHubAppConfig?.gitHubBaseUrl);
+	const alwaysSendBranches = await shouldSendAll("branches-backfill", _jiraHost, logger);
+	const alwaysSendCommits = await shouldSendAll("commits-backfill", _jiraHost, logger);
+	const jiraPayload = transformBranches({ branches, repository }, messagePayload.gitHubAppConfig?.gitHubBaseUrl, alwaysSendBranches, alwaysSendCommits);
 
 	logger.info({ processingTime: Date.now() - startTime, jiraPayloadLength: jiraPayload?.branches?.length }, "Backfill task complete");
 

src/sync/build.ts

@@ -5,19 +5,18 @@ import { GitHubInstallationClient } from "../github/client/github-installation-c
 import { transformWorkflow } from "../transforms/transform-workflow";
 import { GitHubWorkflowPayload } from "~/src/interfaces/github";
 import { transformRepositoryDevInfoBulk } from "~/src/transforms/transform-repository";
-import { numberFlag, NumberFlags } from "config/feature-flags";
+import { numberFlag, NumberFlags, shouldSendAll } from "config/feature-flags";
 import { fetchNextPagesInParallel } from "~/src/sync/parallel-page-fetcher";
 import { BackfillMessagePayload } from "../sqs/sqs.types";
 import { PageSizeAwareCounterCursor } from "~/src/sync/page-counter-cursor";
 
 type BuildWithCursor = { cursor: string } & Octokit.ActionsListRepoWorkflowRunsResponse;
 
 // TODO: add types
-const getTransformedBuilds = async (workflowRun, gitHubInstallationClient, logger) => {
-
+const getTransformedBuilds = async (workflowRun, gitHubInstallationClient, alwaysSend: boolean, logger) => {
 	const transformTasks = workflowRun.map(workflow => {
 		const workflowItem = { workflow_run: workflow, workflow: { id: workflow.id } } as GitHubWorkflowPayload;
-		return transformWorkflow(gitHubInstallationClient, workflowItem, logger);
+		return transformWorkflow(gitHubInstallationClient, workflowItem, alwaysSend, logger);
 	});
 
 	const transformedBuilds = await Promise.all(transformTasks);
@@ -103,7 +102,8 @@ const doGetBuildTask = async (
 	logger.info(`Found ${workflow_runs.length} workflow_runs`);
 	logger.info(`First workflow_run.updated_at=${workflow_runs[0].updated_at}`);
 
-	const builds = await getTransformedBuilds(workflow_runs, gitHubInstallationClient, logger);
+	const alwaysSend = await shouldSendAll("builds-backfill", messagePayload.jiraHost, logger);
+	const builds = await getTransformedBuilds(workflow_runs, gitHubInstallationClient, alwaysSend, logger);
 
 	// When there are no valid builds return early with undefined JiraPayload so that no Jira calls are made
 	if (!builds?.length) {

src/sync/commits.ts

@@ -7,6 +7,7 @@ import { JiraCommitBulkSubmitData } from "src/interfaces/jira";
 import { BackfillMessagePayload } from "~/src/sqs/sqs.types";
 import { TaskResultPayload } from "~/src/sync/sync.types";
 import { createHashWithSharedSecret } from "utils/encryption";
+import { shouldSendAll } from "config/feature-flags";
 
 const fetchCommits = async (gitHubClient: GitHubInstallationClient, repository: Repository, commitSince?: Date, cursor?: string | number, perPage?: number) => {
 	const commitsData = await gitHubClient.getCommitsPage(repository.owner.login, repository.name, perPage, commitSince, cursor);
@@ -40,9 +41,10 @@ export const getCommitTask = async (
 		logger.info(`Last commit authoredDate=${commits[commits.length - 1].authoredDate}`);
 		(logger.fields || {}).commitShaArray = commits.map(c => createHashWithSharedSecret(String(c.oid)));
 	}
-
+	const alwaysSend = await shouldSendAll("commits-backfill", _jiraHost, logger);
 	const jiraPayload = transformCommit(
 		{ commits, repository },
+		alwaysSend,
 		messagePayload.gitHubAppConfig?.gitHubBaseUrl
 	);
 

src/sync/pull-request.ts

@@ -5,7 +5,7 @@ import {
 	transformPullRequest,
 	transformPullRequestRest
 } from "../transforms/transform-pull-request";
-import { statsd }  from "config/statsd";
+import { statsd } from "config/statsd";
 import { metricHttpRequest } from "config/metric-names";
 import { Repository } from "models/subscription";
 import { GitHubInstallationClient } from "../github/client/github-installation-client";
@@ -15,7 +15,7 @@ import { Octokit } from "@octokit/rest";
 import { getCloudOrServerFromHost } from "utils/get-cloud-or-server";
 import { transformRepositoryDevInfoBulk } from "~/src/transforms/transform-repository";
 import { getPullRequestReviews } from "~/src/transforms/util/github-get-pull-request-reviews";
-import { booleanFlag, BooleanFlags, numberFlag, NumberFlags } from "config/feature-flags";
+import { booleanFlag, BooleanFlags, numberFlag, NumberFlags, shouldSendAll } from "config/feature-flags";
 import { isEmpty } from "lodash";
 import { fetchNextPagesInParallel } from "~/src/sync/parallel-page-fetcher";
 import { BackfillMessagePayload } from "../sqs/sqs.types";
@@ -120,9 +120,9 @@ const getPullRequestTaskGraphQL = async (
 
 	const filteredByCreatedSince = response.repository?.pullRequests?.edges
 		.filter(pull => !createdSince || pull.node.createdAt > createdSince.toISOString());
-
+	const alwaysSend = await shouldSendAll("prs-backfill", jiraHost, logger);
 	const pullRequests = filteredByCreatedSince
-		?.map((edge) => transformPullRequest(repository, jiraHost, edge.node, logger))
+		?.map((edge) => transformPullRequest(repository, jiraHost, edge.node, alwaysSend, logger))
 		?.filter((pr) => pr !== undefined) || [];
 
 	(logger.fields || {}).prNumberArray = pullRequests.map(pull => createHashWithSharedSecret(String(pull?.id)));

src/sync/transforms/branch.ts

@@ -14,7 +14,7 @@ import { transformRepositoryDevInfoBulk } from "~/src/transforms/transform-repos
  *  - Title of the last associated Pull Request
  *  - Message from the last commit in that branch
  */
-const mapBranch = (branch, repository) => {
+const mapBranch = (branch, repository, alwaysSend: boolean) => {
 	const branchKeys = jiraIssueKeyParser(branch.name);
 	const pullRequestKeys = jiraIssueKeyParser(
 		branch.associatedPullRequests.nodes.length ? branch.associatedPullRequests.nodes[0].title : ""
@@ -23,7 +23,7 @@ const mapBranch = (branch, repository) => {
 	const allKeys = union(branchKeys, pullRequestKeys, commitKeys)
 		.filter((key) => !!key);
 
-	if (!allKeys.length) {
+	if (!allKeys.length && !alwaysSend) {
 		// If we get here, no issue keys were found anywhere they might be found
 		return undefined;
 	}
@@ -55,10 +55,10 @@ const mapBranch = (branch, repository) => {
  * of commits we got from the GraphQL response and maps the data
  * to the structure needed for the DevInfo API
  */
-const mapCommit = (commit) => {
+const mapCommit = (commit, alwaysSend: boolean) => {
 	const issueKeys = jiraIssueKeyParser(commit.message);
 
-	if (isEmpty(issueKeys)) {
+	if (isEmpty(issueKeys) && !alwaysSend) {
 		return undefined;
 	}
 
@@ -83,16 +83,16 @@ const mapCommit = (commit) => {
  * @param payload
  * @param gitHubBaseUrl - can be undefined for Cloud
  */
-export const transformBranches = (payload: { branches: any, repository: any }, gitHubBaseUrl: string | undefined) => {
+export const transformBranches = (payload: { branches: any, repository: any }, gitHubBaseUrl: string | undefined, alwaysSendBranches: boolean, alwaysSendCommits: boolean) => {
 	// TODO: use reduce instead of map/filter
 	const branches = payload.branches
-		.map((branch) => mapBranch(branch, payload.repository))
+		.map((branch) => mapBranch(branch, payload.repository, alwaysSendBranches))
 		.filter((branch) => !!branch);
 
 	// TODO: use reduce instead of map/filter
 	const commits = payload.branches.flatMap((branch) =>
 		branch.target.history.nodes
-			.map((commit) => mapCommit(commit))
+			.map((commit) => mapCommit(commit, alwaysSendCommits))
 			.filter((branch) => !!branch)
 	);