Merge pull request #2988 from Infisical/daniel/audit-logs-searchability

feat(radar): pagination and filtering

backend/src/ee/routes/v1/secret-scanning-router.ts

@@ -1,9 +1,13 @@
 import { z } from "zod";
 
 import { GitAppOrgSchema, SecretScanningGitRisksSchema } from "@app/db/schemas";
-import { SecretScanningRiskStatus } from "@app/ee/services/secret-scanning/secret-scanning-types";
+import {
+  SecretScanningResolvedStatus,
+  SecretScanningRiskStatus
+} from "@app/ee/services/secret-scanning/secret-scanning-types";
 import { getConfig } from "@app/lib/config/env";
 import { BadRequestError } from "@app/lib/errors";
+import { OrderByDirection } from "@app/lib/types";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
@@ -98,30 +102,95 @@ export const registerSecretScanningRouter = async (server: FastifyZodProvider) =
   });
 
   server.route({
-    url: "/organization/:organizationId/risks",
+    url: "/organization/:organizationId/risks/export",
     method: "GET",
     config: {
       rateLimit: readLimit
     },
     schema: {
       params: z.object({ organizationId: z.string().trim() }),
+      querystring: z.object({
+        repositoryNames: z
+          .string()
+          .optional()
+          .nullable()
+          .transform((val) => (val ? val.split(",") : undefined)),
+        resolvedStatus: z.nativeEnum(SecretScanningResolvedStatus).optional()
+      }),
       response: {
-        200: z.object({ risks: SecretScanningGitRisksSchema.array() })
+        200: z.object({
+          risks: SecretScanningGitRisksSchema.array()
+        })
       }
     },
     onRequest: verifyAuth([AuthMode.JWT]),
     handler: async (req) => {
-      const { risks } = await server.services.secretScanning.getRisksByOrg({
+      const risks = await server.services.secretScanning.getAllRisksByOrg({
         actor: req.permission.type,
         actorId: req.permission.id,
         actorAuthMethod: req.permission.authMethod,
         actorOrgId: req.permission.orgId,
-        orgId: req.params.organizationId
+        orgId: req.params.organizationId,
+        filter: {
+          repositoryNames: req.query.repositoryNames,
+          resolvedStatus: req.query.resolvedStatus
+        }
       });
       return { risks };
     }
   });
 
+  server.route({
+    url: "/organization/:organizationId/risks",
+    method: "GET",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      params: z.object({ organizationId: z.string().trim() }),
+
+      querystring: z.object({
+        offset: z.coerce.number().min(0).default(0),
+        limit: z.coerce.number().min(1).max(20000).default(100),
+        orderBy: z.enum(["createdAt", "name"]).default("createdAt"),
+        orderDirection: z.nativeEnum(OrderByDirection).default(OrderByDirection.DESC),
+        repositoryNames: z
+          .string()
+          .optional()
+          .nullable()
+          .transform((val) => (val ? val.split(",") : undefined)),
+        resolvedStatus: z.nativeEnum(SecretScanningResolvedStatus).optional()
+      }),
+
+      response: {
+        200: z.object({
+          risks: SecretScanningGitRisksSchema.array(),
+          totalCount: z.number(),
+          repos: z.array(z.string())
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { risks, totalCount, repos } = await server.services.secretScanning.getRisksByOrg({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        orgId: req.params.organizationId,
+        filter: {
+          limit: req.query.limit,
+          offset: req.query.offset,
+          orderBy: req.query.orderBy,
+          orderDirection: req.query.orderDirection,
+          repositoryNames: req.query.repositoryNames,
+          resolvedStatus: req.query.resolvedStatus
+        }
+      });
+      return { risks, totalCount, repos };
+    }
+  });
+
   server.route({
     method: "POST",
     url: "/organization/:organizationId/risks/:riskId/status",

backend/src/ee/services/secret-scanning/secret-scanning-dal.ts

@@ -1,9 +1,12 @@
-import { Knex } from "knex";
+import knex, { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
 import { TableName, TSecretScanningGitRisksInsert } from "@app/db/schemas";
-import { DatabaseError } from "@app/lib/errors";
-import { ormify } from "@app/lib/knex";
+import { DatabaseError, GatewayTimeoutError } from "@app/lib/errors";
+import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { OrderByDirection } from "@app/lib/types";
+
+import { SecretScanningResolvedStatus, TGetOrgRisksDTO } from "./secret-scanning-types";
 
 export type TSecretScanningDALFactory = ReturnType<typeof secretScanningDALFactory>;
 
@@ -19,5 +22,70 @@ export const secretScanningDALFactory = (db: TDbClient) => {
     }
   };
 
-  return { ...gitRiskOrm, upsert };
+  const findByOrgId = async (orgId: string, filter: TGetOrgRisksDTO["filter"], tx?: Knex) => {
+    try {
+      // Find statements
+      const sqlQuery = (tx || db.replicaNode())(TableName.SecretScanningGitRisk)
+        // eslint-disable-next-line func-names
+        .where(`${TableName.SecretScanningGitRisk}.orgId`, orgId);
+
+      if (filter.repositoryNames) {
+        void sqlQuery.whereIn(`${TableName.SecretScanningGitRisk}.repositoryFullName`, filter.repositoryNames);
+      }
+
+      if (filter.resolvedStatus) {
+        if (filter.resolvedStatus !== SecretScanningResolvedStatus.All) {
+          const isResolved = filter.resolvedStatus === SecretScanningResolvedStatus.Resolved;
+
+          void sqlQuery.where(`${TableName.SecretScanningGitRisk}.isResolved`, isResolved);
+        }
+      }
+
+      // Select statements
+      void sqlQuery
+        .select(selectAllTableCols(TableName.SecretScanningGitRisk))
+        .limit(filter.limit)
+        .offset(filter.offset);
+
+      if (filter.orderBy) {
+        const orderDirection = filter.orderDirection || OrderByDirection.ASC;
+
+        void sqlQuery.orderBy(filter.orderBy, orderDirection);
+      }
+
+      const countQuery = (tx || db.replicaNode())(TableName.SecretScanningGitRisk)
+        .where(`${TableName.SecretScanningGitRisk}.orgId`, orgId)
+        .count();
+
+      const uniqueReposQuery = (tx || db.replicaNode())(TableName.SecretScanningGitRisk)
+        .where(`${TableName.SecretScanningGitRisk}.orgId`, orgId)
+        .distinct("repositoryFullName")
+        .select("repositoryFullName");
+
+      // we timeout long running queries to prevent DB resource issues (2 minutes)
+      const docs = await sqlQuery.timeout(1000 * 120);
+      const uniqueRepos = await uniqueReposQuery.timeout(1000 * 120);
+      const totalCount = await countQuery;
+
+      return {
+        risks: docs,
+        totalCount: Number(totalCount?.[0].count),
+        repos: uniqueRepos
+          .filter(Boolean)
+          .map((r) => r.repositoryFullName!)
+          .sort((a, b) => a.localeCompare(b))
+      };
+    } catch (error) {
+      if (error instanceof knex.KnexTimeoutError) {
+        throw new GatewayTimeoutError({
+          error,
+          message: "Failed to fetch secret leaks due to timeout. Add more search filters."
+        });
+      }
+
+      throw new DatabaseError({ error });
+    }
+  };
+
+  return { ...gitRiskOrm, upsert, findByOrgId };
 };

backend/src/ee/services/secret-scanning/secret-scanning-service.ts

@@ -15,6 +15,7 @@ import { TSecretScanningDALFactory } from "./secret-scanning-dal";
 import { TSecretScanningQueueFactory } from "./secret-scanning-queue";
 import {
   SecretScanningRiskStatus,
+  TGetAllOrgRisksDTO,
   TGetOrgInstallStatusDTO,
   TGetOrgRisksDTO,
   TInstallAppSessionDTO,
@@ -118,11 +119,21 @@ export const secretScanningServiceFactory = ({
     return Boolean(appInstallation);
   };
 
-  const getRisksByOrg = async ({ actor, orgId, actorId, actorAuthMethod, actorOrgId }: TGetOrgRisksDTO) => {
+  const getRisksByOrg = async ({ actor, orgId, actorId, actorAuthMethod, actorOrgId, filter }: TGetOrgRisksDTO) => {
     const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
     ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.SecretScanning);
+
+    const results = await secretScanningDAL.findByOrgId(orgId, filter);
+
+    return results;
+  };
+
+  const getAllRisksByOrg = async ({ actor, orgId, actorId, actorAuthMethod, actorOrgId }: TGetAllOrgRisksDTO) => {
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.SecretScanning);
+
     const risks = await secretScanningDAL.find({ orgId }, { sort: [["createdAt", "desc"]] });
-    return { risks };
+    return risks;
   };
 
   const updateRiskStatus = async ({
@@ -189,6 +200,7 @@ export const secretScanningServiceFactory = ({
     linkInstallationToOrg,
     getOrgInstallationStatus,
     getRisksByOrg,
+    getAllRisksByOrg,
     updateRiskStatus,
     handleRepoPushEvent,
     handleRepoDeleteEvent

backend/src/ee/services/secret-scanning/secret-scanning-types.ts

@@ -1,4 +1,4 @@
-import { TOrgPermission } from "@app/lib/types";
+import { OrderByDirection, TOrgPermission } from "@app/lib/types";
 
 export enum SecretScanningRiskStatus {
   FalsePositive = "RESOLVED_FALSE_POSITIVE",
@@ -7,6 +7,12 @@ export enum SecretScanningRiskStatus {
   Unresolved = "UNRESOLVED"
 }
 
+export enum SecretScanningResolvedStatus {
+  All = "all",
+  Resolved = "resolved",
+  Unresolved = "unresolved"
+}
+
 export type TInstallAppSessionDTO = TOrgPermission;
 
 export type TLinkInstallSessionDTO = {
@@ -16,7 +22,22 @@ export type TLinkInstallSessionDTO = {
 
 export type TGetOrgInstallStatusDTO = TOrgPermission;
 
-export type TGetOrgRisksDTO = TOrgPermission;
+type RiskFilter = {
+  offset: number;
+  limit: number;
+  orderBy?: "createdAt" | "name";
+  orderDirection?: OrderByDirection;
+  repositoryNames?: string[];
+  resolvedStatus?: SecretScanningResolvedStatus;
+};
+
+export type TGetOrgRisksDTO = {
+  filter: RiskFilter;
+} & TOrgPermission;
+
+export type TGetAllOrgRisksDTO = {
+  filter: Omit<RiskFilter, "offset" | "limit" | "orderBy" | "orderDirection">;
+} & TOrgPermission;
 
 export type TUpdateRiskStatusDTO = {
   riskId: string;

frontend/src/hooks/api/secretScanning/index.tsx

@@ -1,5 +1,6 @@
 export {
   useCreateNewInstallationSession,
+  useExportSecretScanningRisks,
   useLinkGitAppInstallationWithOrg,
   useUpdateRiskStatus
 } from "./mutation";

frontend/src/hooks/api/secretScanning/mutation.ts

@@ -2,7 +2,12 @@ import { useMutation } from "@tanstack/react-query";
 
 import { apiRequest } from "@app/config/request";
 
-import { RiskStatus, TGitAppOrg, TSecretScanningGitRisks } from "./types";
+import {
+  RiskStatus,
+  SecretScanningResolvedStatus,
+  TGitAppOrg,
+  TSecretScanningGitRisks
+} from "./types";
 
 export const useCreateNewInstallationSession = () => {
   return useMutation<{ sessionId: string }, object, { organizationId: string }>({
@@ -43,3 +48,31 @@ export const useLinkGitAppInstallationWithOrg = () => {
     }
   });
 };
+
+export const useExportSecretScanningRisks = () => {
+  return useMutation<
+    TSecretScanningGitRisks[],
+    object,
+    {
+      orgId: string;
+      filter: {
+        repositoryNames?: string[];
+        resolvedStatus?: SecretScanningResolvedStatus;
+      };
+    }
+  >({
+    mutationFn: async ({ filter, orgId }) => {
+      const params = new URLSearchParams({
+        ...(filter.resolvedStatus && { resolvedStatus: filter.resolvedStatus }),
+        ...(filter.repositoryNames && { repositoryNames: filter.repositoryNames.join(",") })
+      });
+
+      const { data } = await apiRequest.get<{
+        risks: TSecretScanningGitRisks[];
+      }>(`/api/v1/secret-scanning/organization/${orgId}/risks/export`, {
+        params
+      });
+      return data.risks;
+    }
+  });
+};