diff --git a/.env.example b/.env.example index f8c910f..d2d7123 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,15 @@ ACCESS_KEY=tenable-access-key SECRET_KEY=tenable-secret-key ASSET_API_TIMEOUT_IN_MINUTES=30 + +# Configuration filters for Vulnerabilities VULNERABILITY_API_TIMEOUT_IN_MINUTES=30 VULNERABILITY_SEVERITIES=info,low,medium,high,critical VULNERABILITY_STATES=open,reopened,fixed + +# Configuration filters for Compliance Findings +COMPLIANCE_LAST_SEEN=15,30,60,90 +COMPLIANCE_STATE=OPEN,REOPENED,FIXED +COMPLIANCE_RESULT=PASSED,FAILED,WARNING,SKIPPED,UNKNOWN,ERROR +COMPLIANCE_NUM_FINDINGS=10000 + diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1f282..baed044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -300,7 +300,7 @@ cvss3Vector - cvssVector - hasPatch ## [8.5.1] 2022-08-05 -- fix tenable_asset `firstSeen` and `lastSeen` properties to be human-readable +- fix tenable_asset `firstSeen` and `complianceLastSeen` properties to be human-readable ## [8.5.0] 2022-06-08 diff --git a/docs/jupiterone.md b/docs/jupiterone.md index ff4c06c..aba9464 100644 --- a/docs/jupiterone.md +++ b/docs/jupiterone.md @@ -90,6 +90,7 @@ The following entities are created: | Account | `tenable_account` | `Account` | | Agent | `tenable_agent` | `HostAgent` | | Asset | `tenable_asset` | `Record` | +| Compliance Finding | `tenable_compliance_finding` | `Finding` | | Container Finding | `tenable_container_finding` | `Finding` | | Container Image | `tenable_container_image` | `Image` | | Container Malware | `tenable_container_malware` | `Finding` | @@ -113,6 +114,7 @@ The following relationships are created: | `tenable_account` | **HAS** | `tenable_user` | | `tenable_account` | **PROVIDES** | `tenable_scanner` | | `tenable_agent` | **PROTECTS** | `tenable_asset` | +| `tenable_asset` | **HAS** | `tenable_compliance_finding` | | `tenable_asset` | **HAS** | `tenable_vulnerability_finding` | | `tenable_container_image` | **HAS** | `tenable_container_finding` | | `tenable_container_image` | **HAS** | `tenable_container_malware` | diff --git a/src/config.ts b/src/config.ts index 507765e..f515087 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,9 +7,14 @@ export interface IntegrationConfig extends IntegrationInstanceConfig { accessKey: string; secretKey: string; vulnerabilityApiTimeoutInMinutes?: number; + complianceApiTimeoutInMinutes?: number; assetApiTimeoutInMinutes?: number; vulnerabilitySeverities?: string; vulnerabilityStates?: string; + complianceLastSeen?: string; + complianceState?: string; + complianceResult?: string; + complianceNumFindings?: number; } export const instanceConfigFields: IntegrationInstanceConfigFieldMap = { @@ -32,4 +37,19 @@ export const instanceConfigFields: IntegrationInstanceConfigFieldMap = { vulnerabilityStates: { type: 'string', }, + complianceLastSeen: { + type: 'string', + }, + complianceApiTimeoutInMinutes: { + type: 'string', + }, + complianceState: { + type: 'string', + }, + complianceResult: { + type: 'string', + }, + complianceNumFindings: { + type: 'string', + }, }; diff --git a/src/index.ts b/src/index.ts index 97b9c34..ac361a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import { IntegrationConfig, instanceConfigFields } from './config'; import validateInvocation from './invocationValidator'; import { integrationSteps } from './steps'; import getStepStartStates from './getStepStartStates'; +import { ingestionConfig } from './ingestionConfig'; export const invocationConfig: IntegrationInvocationConfig = { @@ -10,4 +11,5 @@ export const invocationConfig: IntegrationInvocationConfig = validateInvocation, integrationSteps, getStepStartStates, + ingestionConfig, }; diff --git a/src/ingestionConfig.ts b/src/ingestionConfig.ts new file mode 100644 index 0000000..9a218da --- /dev/null +++ b/src/ingestionConfig.ts @@ -0,0 +1,60 @@ +import { IntegrationIngestionConfigFieldMap } from '@jupiterone/integration-sdk-core'; +import { INGESTION_SOURCE_IDS } from './steps/constants'; + +export const ingestionConfig: IntegrationIngestionConfigFieldMap = { + [INGESTION_SOURCE_IDS.ACCOUNT]: { + title: 'Account', + description: 'Tenable accounts', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.SERVICE]: { + title: 'Service', + description: 'Service descriptions', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.ASSETS]: { + title: 'Assets', + description: 'Asset descriptions', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.VULNERABILITIES]: { + title: 'Vulnerabilities', + description: 'Vulnerability descriptions', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.USERS]: { + title: 'Users', + description: 'User information', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.CONTAINER_IMAGES]: { + title: 'Container Images', + description: 'Container image descriptions', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.CONTAINER_REPOSITORIES]: { + title: 'Container Repositories', + description: 'Container repository descriptions', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.CONTAINER_REPORTS]: { + title: 'Container Reports', + description: 'Reports on container statuses', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.SCANNER_IDS]: { + title: 'Scanner IDs', + description: 'Scanner ID information', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.AGENTS]: { + title: 'Agents', + description: 'Agent information', + defaultsToDisabled: false, + }, + [INGESTION_SOURCE_IDS.COMPLIANCE_FINDINGS]: { + title: 'Compliance Findings', + description: 'Compliance findings', + defaultsToDisabled: true, + }, +}; diff --git a/src/invocationValidator.ts b/src/invocationValidator.ts index 9e5cdab..17294b9 100644 --- a/src/invocationValidator.ts +++ b/src/invocationValidator.ts @@ -9,6 +9,8 @@ import TenableClient from './tenable/TenableClient'; import { VALID_VULNERABILITY_STATES, VALID_VULNERABILITY_SEVERITIES, + VALID_COMPLIANCE_STATES, + VALID_COMPLIANCE_RESULT, } from './tenable/client'; import { toNum } from './utils/dataType'; @@ -50,6 +52,38 @@ function validateVulnerabilityStates(states: string) { } } +function validateComplianceStates(states: string) { + const statesValues = states.replace(/\s+/g, '').split(','); + for (const state of statesValues) { + if (!(VALID_COMPLIANCE_STATES as unknown as string[]).includes(state)) { + throw new IntegrationValidationError( + `States - ${state} - is not valid. Valid Compliance states include ${VALID_COMPLIANCE_STATES.map( + (v) => v, + )}`, + ); + } + } +} + +function validateComplianceResults(complianceResult: string) { + const complianceResultValues = complianceResult + .replace(/\s+/g, '') + .split(','); + for (const complianceResult of complianceResultValues) { + if ( + !(VALID_COMPLIANCE_RESULT as unknown as string[]).includes( + complianceResult, + ) + ) { + throw new IntegrationValidationError( + `complianceResult - ${complianceResult} - is not valid. Valid complianceResult include ${VALID_COMPLIANCE_RESULT.map( + (v) => v, + )}`, + ); + } + } +} + /** * Performs validation of the execution before the execution handler function is * invoked. @@ -118,6 +152,34 @@ export default async function validateInvocation( validateVulnerabilityStates(vulnerabilityStates); } + if (config.complianceStates) { + const complianceStates = + (executionContext.instance.config.complianceStates = + typeof config.complianceStates === 'string' + ? config.complianceStates.replace(/\s+/g, '') + : config.complianceStates); + validateComplianceStates(complianceStates); + } + + if (config.complianceResult) { + const complianceResult = + (executionContext.instance.config.complianceResult = + typeof config.complianceResult === 'string' + ? config.complianceResult.replace(/\s+/g, '') + : config.complianceResult); + validateComplianceResults(complianceResult); + } + + if (config.complianceNumFindings) { + const numFindings = Number(config.complianceNumFindings); + if (isNaN(numFindings) || numFindings < 50 || numFindings > 10000) { + throw new IntegrationConfigLoadError( + `'numFindings' config value is invalid (val=${numFindings}, min=50, max=10000)`, + ); + } + executionContext.instance.config.complianceNumFindings = numFindings; + } + const provider = new TenableClient({ logger, accessToken: config.accessKey, diff --git a/src/steps/access/index.ts b/src/steps/access/index.ts index f46ccd1..839e5a8 100644 --- a/src/steps/access/index.ts +++ b/src/steps/access/index.ts @@ -3,7 +3,12 @@ import { Step, } from '@jupiterone/integration-sdk-core'; import { IntegrationConfig } from '../../config'; -import { Entities, Relationships, StepIds } from '../constants'; +import { + Entities, + INGESTION_SOURCE_IDS, + Relationships, + StepIds, +} from '../constants'; import { getAccount } from '../account/util'; import TenableClient from '../../tenable/TenableClient'; import { createAccountUserRelationship, createUserEntity } from './converters'; @@ -35,6 +40,7 @@ export const userStep: Step< id: StepIds.USERS, name: 'Fetch Users', entities: [Entities.USER], + ingestionSourceId: INGESTION_SOURCE_IDS.USERS, relationships: [Relationships.ACCOUNT_HAS_USER], dependsOn: [StepIds.ACCOUNT], executionHandler: fetchUsers, diff --git a/src/steps/account/index.ts b/src/steps/account/index.ts index 52e55f8..03aa89c 100644 --- a/src/steps/account/index.ts +++ b/src/steps/account/index.ts @@ -3,7 +3,7 @@ import { Step, } from '@jupiterone/integration-sdk-core'; import { IntegrationConfig } from '../../config'; -import { Entities, StepIds } from '../constants'; +import { Entities, INGESTION_SOURCE_IDS, StepIds } from '../constants'; import { createAccountEntity } from './converters'; import { getAccount } from './util'; @@ -19,6 +19,7 @@ export const accountStep: Step< id: StepIds.ACCOUNT, name: 'Fetch Account', entities: [Entities.ACCOUNT], + ingestionSourceId: INGESTION_SOURCE_IDS.ACCOUNT, relationships: [], dependsOn: [], executionHandler: fetchAccount, diff --git a/src/steps/agents/index.ts b/src/steps/agents/index.ts index 14e7819..cf3b971 100644 --- a/src/steps/agents/index.ts +++ b/src/steps/agents/index.ts @@ -5,7 +5,12 @@ import { Step, createDirectRelationship, } from '@jupiterone/integration-sdk-core'; -import { Entities, StepIds, Relationships } from '../constants'; +import { + Entities, + StepIds, + Relationships, + INGESTION_SOURCE_IDS, +} from '../constants'; import { IntegrationConfig } from '../../config'; import { DATA_SCANNER_IDS } from '../scanners/constants'; import { createAgentEntity } from './converters'; @@ -125,6 +130,7 @@ export const agentsSteps: Step< id: StepIds.AGENTS, name: 'Fetch Agents', entities: [Entities.AGENT], + ingestionSourceId: INGESTION_SOURCE_IDS.AGENTS, relationships: [Relationships.ACCOUNT_HAS_AGENT], dependsOn: [StepIds.ACCOUNT, StepIds.SCANNER_IDS], executionHandler: fetchAgents, @@ -133,6 +139,7 @@ export const agentsSteps: Step< id: StepIds.AGENT_RELATIONSHIPS, name: 'Build Host Agent Protects Agents Relationship', entities: [], + ingestionSourceId: INGESTION_SOURCE_IDS.AGENTS, relationships: [Relationships.HOSTAGENT_PROTECTS_DEVICE], dependsOn: [StepIds.ASSETS, StepIds.AGENTS], executionHandler: buildAgentRelationships, diff --git a/src/steps/compliance-finding/converter.ts b/src/steps/compliance-finding/converter.ts new file mode 100644 index 0000000..8bc47b3 --- /dev/null +++ b/src/steps/compliance-finding/converter.ts @@ -0,0 +1,38 @@ +import { Entities } from '../constants'; +import { + createIntegrationEntity, + Entity, +} from '@jupiterone/integration-sdk-core'; +import { generateEntityKey } from '../../utils/generateKey'; + +export function createComplianceFindingEntity(complianceChunk): Entity { + return createIntegrationEntity({ + entityData: { + source: complianceChunk, + assign: { + _class: Entities.COMPLIANCE_FINDINGS._class, + _type: Entities.COMPLIANCE_FINDINGS._type, + _key: generateEntityKey( + Entities.COMPLIANCE_FINDINGS._type, + complianceChunk.uuid, + ), + + // Schema required fields. + category: ['network', 'host'], + severity: ['low', 'medium'], + numericSeverity: [1, 2], + id: String(complianceChunk.id), + agentId: complianceChunk.id, + displayName: complianceChunk.name, + open: complianceChunk.state === 'OPEN', + + // Entity additional data. + name: complianceChunk.name, + status: complianceChunk.status, + firstSeen: complianceChunk.first_seen, + lastSeen: complianceChunk.last_seen, + agentName: complianceChunk.agent_name, + }, + }, + }); +} diff --git a/src/steps/compliance-finding/filters.ts b/src/steps/compliance-finding/filters.ts new file mode 100644 index 0000000..f6816c0 --- /dev/null +++ b/src/steps/compliance-finding/filters.ts @@ -0,0 +1,56 @@ +import { IntegrationConfig } from '../../config'; +import { + ExportComplianceFindingsFilter, + complianceChunkState, + complianceChunkResult, +} from '../../tenable/client'; +import { subDays, getUnixTime } from 'date-fns'; + +const DEFAULT_STATES: complianceChunkState[] = ['OPEN', 'REOPENED', 'FIXED']; +const DEFAULT_RESULTS: complianceChunkResult[] = [ + 'PASSED', + 'FAILED', + 'WARNING', + 'SKIPPED', + 'UNKNOWN', + 'ERROR', +]; +const DEFAULT_LAST_SEEN_DAYS = 30; // Default to 30 days if not provided + +function parseComplianceStates(states: string): complianceChunkState[] { + return states.split(',') as complianceChunkState[]; +} + +function parseComplianceResults(results: string): complianceChunkResult[] { + return results.split(',') as complianceChunkResult[]; +} + +function calculateLastSeenTimestamp(daysAgo: number): number { + const lastSeenDate = subDays(new Date(), daysAgo); + return getUnixTime(lastSeenDate); +} + +export function buildComplianceFilters( + config: IntegrationConfig, +): ExportComplianceFindingsFilter { + const lastSeenDays = config.complianceLastSeen + ? Number(config.complianceLastSeen) + : DEFAULT_LAST_SEEN_DAYS; + if (isNaN(lastSeenDays)) { + throw new Error( + `Invalid complianceLastSeen value: ${config.complianceLastSeen}`, + ); + } + + const lastSeenTimestamp = calculateLastSeenTimestamp(lastSeenDays); + + return { + state: config.complianceState + ? parseComplianceStates(config.complianceState) + : DEFAULT_STATES, + compliance_results: config.complianceResults + ? parseComplianceResults(config.complianceResults) + : DEFAULT_RESULTS, + last_seen: lastSeenTimestamp, + }; +} diff --git a/src/steps/compliance-finding/index.test.ts b/src/steps/compliance-finding/index.test.ts new file mode 100644 index 0000000..582db21 --- /dev/null +++ b/src/steps/compliance-finding/index.test.ts @@ -0,0 +1,54 @@ +jest.setTimeout(50000); + +import { StepIds } from '../constants'; +import { buildStepTestConfig } from '../../../test/config'; +import { executeStepWithDependencies } from '@jupiterone/integration-sdk-testing'; +import { setupTenableRecording, Recording } from '../../../test/recording'; + +let recording: Recording; + +afterEach(async () => { + if (recording) { + await recording.stop(); + } +}); + +describe.skip('step-compliance-findings', () => { + test('success', async () => { + recording = setupTenableRecording({ + name: 'step-compliance-findings', + directory: __dirname, + options: { + recordFailedRequests: false, + matchRequestsBy: { + order: true, + }, + }, + }); + + const stepConfig = buildStepTestConfig(StepIds.COMPLIANCE_FINDINGS); + const stepResults = await executeStepWithDependencies(stepConfig); + expect(stepResults).toMatchStepMetadata(stepConfig); + }); +}); + +describe.skip('build-asset-compliance-findings-relationships', () => { + test('success', async () => { + recording = setupTenableRecording({ + name: 'build-asset-compliance-findings-relationships', + directory: __dirname, + options: { + recordFailedRequests: false, + matchRequestsBy: { + order: true, + }, + }, + }); + + const stepConfig = buildStepTestConfig( + StepIds.ASSET_COMPLIANCE_FINDINGS_RELATIONSHIPS, + ); + const stepResults = await executeStepWithDependencies(stepConfig); + expect(stepResults).toMatchStepMetadata(stepConfig); + }); +}); diff --git a/src/steps/compliance-finding/index.ts b/src/steps/compliance-finding/index.ts new file mode 100644 index 0000000..7572801 --- /dev/null +++ b/src/steps/compliance-finding/index.ts @@ -0,0 +1,127 @@ +import { + IntegrationStepExecutionContext, + RelationshipClass, + Step, + createDirectRelationship, + getRawData, +} from '@jupiterone/integration-sdk-core'; +import TenableClient from '../../tenable/TenableClient'; +import { + Entities, + StepIds, + Relationships, + INGESTION_SOURCE_IDS, +} from '../constants'; +import { IntegrationConfig } from '../../config'; +import { createComplianceFindingEntity } from './converter'; +import { buildComplianceFilters } from './filters'; + +export async function fetchComplianceFindings( + context: IntegrationStepExecutionContext, +): Promise { + const { jobState, logger, instance } = context; + const { complianceApiTimeoutInMinutes, accessKey, secretKey } = + instance.config; + + const provider = new TenableClient({ + logger: logger, + accessToken: accessKey, + secretToken: secretKey, + }); + + logger.info( + { complianceApiTimeoutInMinutes }, + 'Attempting to fetch compliance findings...', + ); + + let duplicateKeysEncountered = 0; + await provider.iterateComplianceData( + async (finding) => { + const complianceEntity = createComplianceFindingEntity(finding); + if (jobState.hasKey(complianceEntity._key)) { + logger.debug( + { + _key: complianceEntity._key, + }, + 'Debug: duplicate tenable_compliance_finding _key encountered', + ); + duplicateKeysEncountered += 1; + } + try { + await jobState.addEntity(complianceEntity); + } catch (error) { + /* Empty for now, will remove try/catch when we have a report of duplicated keys */ + } + }, + { + timeoutInMinutes: complianceApiTimeoutInMinutes, + exportComplianceFindingsOptions: { + num_findings: 5000, + filters: buildComplianceFilters(instance.config), + }, + }, + ); + + if (duplicateKeysEncountered > 0) { + logger.info( + { duplicateKeysEncountered }, + `Found duplicate keys for "tenable_compliance_finding" entity`, + ); + } +} + +export async function buildAssetComplianceFindingRelationships( + context: IntegrationStepExecutionContext, +): Promise { + const { jobState } = context; + await jobState.iterateEntities( + { _type: Entities.COMPLIANCE_FINDINGS._type }, + async (complianceEntity) => { + const complianceData = getRawData(complianceEntity); + if (complianceData && complianceData.asset && complianceData.asset.id) { + const assetKey = complianceData.asset.id; + + if (!assetKey) { + context.logger.warn( + `Cannot build Relationship. Error: Missing Key. assetKey: ${assetKey}`, + ); + + return; + } + + await jobState.addRelationship( + createDirectRelationship({ + _class: RelationshipClass.HAS, + fromKey: assetKey, + fromType: Entities.ASSET._type, + toKey: complianceEntity._key, + toType: Entities.COMPLIANCE_FINDINGS._type, + }), + ); + } + }, + ); +} + +export const complianceFindingSteps: Step< + IntegrationStepExecutionContext +>[] = [ + { + id: StepIds.COMPLIANCE_FINDINGS, + name: 'Fetch Compliance', + entities: [Entities.COMPLIANCE_FINDINGS], + ingestionSourceId: INGESTION_SOURCE_IDS.COMPLIANCE_FINDINGS, + relationships: [], + dependsOn: [], + executionHandler: fetchComplianceFindings, + }, + { + id: StepIds.ASSET_COMPLIANCE_FINDINGS_RELATIONSHIPS, + name: 'Build Asset Has Compliance Finding Relationship', + entities: [], + ingestionSourceId: INGESTION_SOURCE_IDS.COMPLIANCE_FINDINGS, + relationships: [Relationships.ASSET_HAS_COMPLIANCE_FINDINGS], + dependsOn: [StepIds.ASSETS, StepIds.COMPLIANCE_FINDINGS], + executionHandler: buildAssetComplianceFindingRelationships, + }, +]; diff --git a/src/steps/constants.ts b/src/steps/constants.ts index 15be8be..62911d0 100644 --- a/src/steps/constants.ts +++ b/src/steps/constants.ts @@ -8,6 +8,8 @@ import { export const SERVICE_ENTITY_DATA_KEY = 'entity:service'; +export const SLEEP_TIME = 60_000; + export const StepIds = { ACCOUNT: 'step-account', SERVICE: 'step-service', @@ -23,6 +25,9 @@ export const StepIds = { SCANNER_IDS: 'step-scanner-ids', AGENTS: 'step-agents', AGENT_RELATIONSHIPS: 'build-agent-relationships', + COMPLIANCE_FINDINGS: 'step-compliance-findings', + ASSET_COMPLIANCE_FINDINGS_RELATIONSHIPS: + 'build-asset-compliance-findings-relationships', }; export const Entities: Record< @@ -37,7 +42,8 @@ export const Entities: Record< | 'CONTAINER_UNWANTED_PROGRAM' | 'VULNERABILITY' | 'USER' - | 'AGENT', + | 'AGENT' + | 'COMPLIANCE_FINDINGS', StepEntityMetadata > = { ACCOUNT: { @@ -101,6 +107,11 @@ export const Entities: Record< _class: ['HostAgent'], _type: 'tenable_agent', }, + COMPLIANCE_FINDINGS: { + resourceName: 'Compliance Finding', + _class: ['Finding'], + _type: 'tenable_compliance_finding', + }, }; export const Relationships: Record< @@ -120,7 +131,8 @@ export const Relationships: Record< | 'REPORT_IDENTIFIED_UNWANTED_PROGRAM' | 'ASSET_HAS_VULN' | 'ACCOUNT_HAS_AGENT' - | 'HOSTAGENT_PROTECTS_DEVICE', + | 'HOSTAGENT_PROTECTS_DEVICE' + | 'ASSET_HAS_COMPLIANCE_FINDINGS', StepRelationshipMetadata > = { ACCOUNT_HAS_USER: { @@ -265,11 +277,17 @@ export const Relationships: Record< targetType: Entities.AGENT._type, }, HOSTAGENT_PROTECTS_DEVICE: { - _type: 'tenable_agent_has_asset', + _type: 'tenable_agent_protects_asset', sourceType: Entities.AGENT._type, _class: RelationshipClass.PROTECTS, targetType: Entities.ASSET._type, }, + ASSET_HAS_COMPLIANCE_FINDINGS: { + _type: 'tenable_asset_has_compliance_finding', + sourceType: Entities.ASSET._type, + _class: RelationshipClass.HAS, + targetType: Entities.COMPLIANCE_FINDINGS._type, + }, }; export const MappedRelationships: Record< @@ -353,3 +371,17 @@ export const MappedRelationships: Record< }, }, }; + +export const INGESTION_SOURCE_IDS = { + ACCOUNT: 'account', + SERVICE: 'service', + ASSETS: 'assets', + VULNERABILITIES: 'vulnerabilities', + USERS: 'users', + CONTAINER_IMAGES: 'container-images', + CONTAINER_REPOSITORIES: 'container-repositories', + CONTAINER_REPORTS: 'container-reports', + SCANNER_IDS: 'scanner-ids', + AGENTS: 'agents', + COMPLIANCE_FINDINGS: 'compliance-findings', +}; diff --git a/src/steps/containers/index.ts b/src/steps/containers/index.ts index 94edb18..0148da2 100644 --- a/src/steps/containers/index.ts +++ b/src/steps/containers/index.ts @@ -3,7 +3,12 @@ import { Step, } from '@jupiterone/integration-sdk-core'; import { IntegrationConfig } from '../../config'; -import { Entities, Relationships, StepIds } from '../constants'; +import { + Entities, + INGESTION_SOURCE_IDS, + Relationships, + StepIds, +} from '../constants'; import { buildRepositoryImagesRelationship, fetchContainerImages, @@ -18,6 +23,7 @@ export const containerSteps: Step< id: StepIds.CONTAINER_REPOSITORIES, name: 'Fetch Container Repositories', entities: [Entities.CONTAINER_REPOSITORY], + ingestionSourceId: INGESTION_SOURCE_IDS.CONTAINER_REPOSITORIES, relationships: [Relationships.ACCOUNT_HAS_CONTAINER_REPOSITORY], dependsOn: [StepIds.ACCOUNT], executionHandler: fetchContainerRepositories, @@ -26,6 +32,7 @@ export const containerSteps: Step< id: StepIds.CONTAINER_IMAGES, name: 'Fetch Container Images', entities: [Entities.CONTAINER_IMAGE], + ingestionSourceId: INGESTION_SOURCE_IDS.CONTAINER_IMAGES, relationships: [ Relationships.ACCOUNT_HAS_CONTAINER_IMAGE, Relationships.SERVICE_SCANS_CONTAINER_IMAGE, @@ -37,6 +44,7 @@ export const containerSteps: Step< id: StepIds.REPOSITORY_IMAGES_RELATIONSHIPS, name: 'Build Repository Images Relationships', entities: [], + ingestionSourceId: INGESTION_SOURCE_IDS.CONTAINER_IMAGES, relationships: [Relationships.CONTAINER_REPOSITORY_HAS_IMAGE], dependsOn: [StepIds.CONTAINER_IMAGES, StepIds.CONTAINER_REPOSITORIES], executionHandler: buildRepositoryImagesRelationship, @@ -50,6 +58,7 @@ export const containerSteps: Step< Entities.CONTAINER_MALWARE, Entities.CONTAINER_UNWANTED_PROGRAM, ], + ingestionSourceId: INGESTION_SOURCE_IDS.CONTAINER_REPORTS, relationships: [ Relationships.CONTAINER_IMAGE_HAS_REPORT, Relationships.CONTAINER_IMAGE_HAS_FINDING, diff --git a/src/steps/index.ts b/src/steps/index.ts index 9c4c64d..3016867 100644 --- a/src/steps/index.ts +++ b/src/steps/index.ts @@ -1,6 +1,7 @@ import { accountStep } from './account'; import { containerSteps } from './containers'; import { scanSteps } from './vulnerabilities'; +import { complianceFindingSteps } from './compliance-finding'; import { userStep } from './access'; import { serviceSteps } from './service'; import { scannerStep } from './scanners'; @@ -16,4 +17,5 @@ export const integrationSteps: IntegrationStep[] = [ userStep, scannerStep, ...agentsSteps, + ...complianceFindingSteps, ]; diff --git a/src/steps/scanners/index.ts b/src/steps/scanners/index.ts index 108e3e5..9ed0453 100644 --- a/src/steps/scanners/index.ts +++ b/src/steps/scanners/index.ts @@ -3,7 +3,7 @@ import { IntegrationStepExecutionContext, Step, } from '@jupiterone/integration-sdk-core'; -import { StepIds } from '../constants'; +import { INGESTION_SOURCE_IDS, StepIds } from '../constants'; import { IntegrationConfig } from '../../config'; import { DATA_SCANNER_IDS } from './constants'; @@ -34,6 +34,7 @@ export const scannerStep: Step< id: StepIds.SCANNER_IDS, name: 'Fetch Scanner IDs', entities: [], + ingestionSourceId: INGESTION_SOURCE_IDS.SCANNER_IDS, relationships: [], dependsOn: [], executionHandler: fetchScannerIds, diff --git a/src/steps/service/index.ts b/src/steps/service/index.ts index 3a139af..ea1e2f9 100644 --- a/src/steps/service/index.ts +++ b/src/steps/service/index.ts @@ -5,6 +5,7 @@ import { import { IntegrationConfig } from '../../config'; import { Entities, + INGESTION_SOURCE_IDS, Relationships, SERVICE_ENTITY_DATA_KEY, StepIds, @@ -42,6 +43,7 @@ export const serviceSteps: Step< id: StepIds.SERVICE, name: 'Fetch Service Details', entities: [Entities.SERVICE], + ingestionSourceId: INGESTION_SOURCE_IDS.SERVICE, relationships: [Relationships.ACCOUNT_PROVIDES_SERVICE], dependsOn: [StepIds.ACCOUNT], executionHandler: fetchServiceDetails, diff --git a/src/steps/vulnerabilities/index.ts b/src/steps/vulnerabilities/index.ts index 4489c0d..b1ade44 100644 --- a/src/steps/vulnerabilities/index.ts +++ b/src/steps/vulnerabilities/index.ts @@ -8,6 +8,7 @@ import { import { IntegrationConfig } from '../../config'; import { Entities, + INGESTION_SOURCE_IDS, MappedRelationships, Relationships, StepIds, @@ -265,6 +266,7 @@ export const scanSteps: Step< id: StepIds.ASSETS, name: 'Fetch Assets', entities: [Entities.ASSET], + ingestionSourceId: INGESTION_SOURCE_IDS.ASSETS, relationships: [Relationships.ACCOUNT_HAS_ASSET], mappedRelationships: [ MappedRelationships.TENABLE_ASSET_IS_AWS_INSTANCE, @@ -278,6 +280,7 @@ export const scanSteps: Step< id: StepIds.VULNERABILITIES, name: 'Fetch Vulnerabilities', entities: [Entities.VULNERABILITY], + ingestionSourceId: INGESTION_SOURCE_IDS.VULNERABILITIES, relationships: [], dependsOn: [], executionHandler: fetchVulnerabilities, @@ -286,6 +289,7 @@ export const scanSteps: Step< id: StepIds.ASSET_VULNERABILITY_RELATIONSHIPS, name: 'Build Asset -> Vulnerability Relationships', entities: [], + ingestionSourceId: INGESTION_SOURCE_IDS.ASSETS, relationships: [Relationships.ASSET_HAS_VULN], mappedRelationships: [ MappedRelationships.VULNERABILITY_HAS_AWS_INSTANCE, @@ -299,6 +303,7 @@ export const scanSteps: Step< id: StepIds.VULNERABILITY_CVE_RELATIONSHIPS, name: 'Build Vulnerability -> CVE Mapped Relationships', entities: [], + ingestionSourceId: INGESTION_SOURCE_IDS.VULNERABILITIES, relationships: [], mappedRelationships: [MappedRelationships.VULNERABILITY_IS_CVE], dependsOn: [StepIds.VULNERABILITIES], diff --git a/src/tenable/TenableClient.ts b/src/tenable/TenableClient.ts index 57726b1..d45b14b 100644 --- a/src/tenable/TenableClient.ts +++ b/src/tenable/TenableClient.ts @@ -1,8 +1,12 @@ import { version as graphTenablePackageVersion } from '../../package.json'; import Client, { Agent, + ComplianceChunk, + ComplianceExportStatusResponse, + ComplianceUuid, ContainerImage, ContainerRepository, + ExportComplianceFindingsOptions, ExportStatus, Scanner, TenableResponse, @@ -33,6 +37,7 @@ import { sleep } from '@lifeomic/attempt'; import pMap from 'p-map'; import { addMinutes, getUnixTime, isAfter, sub } from 'date-fns'; import { paginated } from '../utils/pagination'; +import { SLEEP_TIME } from '../steps/constants'; function length(resources?: any[]): number { return resources ? resources.length : 0; @@ -90,6 +95,138 @@ export default class TenableClient { return usersResponse.users; } + private async cancelComplianceExport( + exportUuid: string, + ): Promise { + const cancelExportResponse = await this.retryRequest(() => + this.client.cancelComplianceFindingExport(exportUuid), + ); + + this.logger.info( + { + cancelExportResponse, + }, + 'Cancelled Tenable Compliance export', + ); + + return cancelExportResponse; + } + + private async fetchComplianceFinding( + options: ExportComplianceFindingsOptions, + ): Promise { + const complianceExportResponse = await this.retryRequest(() => + this.client.exportComplianceData(options), + ); + + this.logger.info( + { + options, + complianceExportResponse, + }, + 'Started Complaince Finding export', + ); + return complianceExportResponse; + } + + public async fetchComplianceExportStatus( + exportUuid: string, + ): Promise { + const exportStatusResponse = await this.retryRequest(() => + this.client.fetchComplianceStatus(exportUuid), + ); + this.logger.info( + { + exportUuid, + exportStatusResponse, + }, + 'Fetched Tenable Compliance export status', + ); + return exportStatusResponse; + } + + private async fetchComplianceExportChunk( + exportUuid: string, + chunkId: number, + ): Promise { + const complianceChunkResponse = await this.retryRequest(() => + this.client.fetchComplianceChunk(exportUuid, chunkId), + ); + + this.logger.info( + { + exportUuid, + chunkId, + vulnerabilitiesExportResponse: complianceChunkResponse.length, + }, + 'Fetched Tenable Compliance export chunk', + ); + return complianceChunkResponse; + } + + public async iterateComplianceData( + callback: (compliance: ComplianceChunk) => void | Promise, + options?: { + timeoutInMinutes?: number; + exportComplianceFindingsOptions?: ExportComplianceFindingsOptions; + }, + ) { + const exportComplianceFindingsOptions = + options?.exportComplianceFindingsOptions || { + num_findings: 5000, + filters: { + last_seen: getUnixTime(sub(Date.now(), { days: 30 })), + state: ['OPEN', 'REOPENED', 'FIXED'], + compliance_results: [ + 'PASSED', + 'FAILED', + 'WARNING', + 'SKIPPED', + 'UNKNOWN', + 'ERROR', + ], + }, + }; + + const timeoutInMinutes = options?.timeoutInMinutes || 180; + const { export_uuid: exportUuid } = await this.fetchComplianceFinding( + exportComplianceFindingsOptions, + ); + + let { status, chunks_available: chunksAvailable } = + await this.fetchComplianceExportStatus(exportUuid); + + const timeLimit = addMinutes(Date.now(), timeoutInMinutes); + while ([ExportStatus.Processing, ExportStatus.Queued].includes(status)) { + if (isAfter(Date.now(), timeLimit)) { + await this.cancelComplianceExport(exportUuid); + throw new IntegrationError({ + code: 'TenableClientApiError', + message: `Compliance Finding export ${exportUuid} failed to finish processing in time limit`, + }); + } + + ({ status, chunks_available: chunksAvailable } = + await this.fetchComplianceExportStatus(exportUuid)); + await sleep(SLEEP_TIME); // Sleep 60 seconds between status checks. + } + + await pMap( + chunksAvailable, + async (chunkId) => { + const complianceChunks = await this.fetchComplianceExportChunk( + exportUuid, + chunkId, + ); + for (const complianceChunk of complianceChunks) { + await callback(complianceChunk); + } + }, + { concurrency: 3 }, + ); + return { exportUuid }; + } + private async exportVulnerabilities( options: ExportVulnerabilitiesOptions, ): Promise { @@ -196,7 +333,7 @@ export default class TenableClient { ({ status, chunks_available: chunksAvailable } = await this.fetchVulnerabilitiesExportStatus(exportUuid)); - await sleep(60_000); // Sleep 60 seconds between status checks. + await sleep(SLEEP_TIME); // Sleep 60 seconds between status checks. } await pMap( @@ -315,7 +452,7 @@ export default class TenableClient { ({ status, chunks_available: chunksAvailable } = await this.fetchAssetsExportStatus(exportUuid)); - await sleep(60_000); // Sleep 60 seconds between status checks. + await sleep(SLEEP_TIME); // Sleep 60 seconds between status checks. } await pMap( diff --git a/src/tenable/client/index.ts b/src/tenable/client/index.ts index 5e6ff78..f081809 100644 --- a/src/tenable/client/index.ts +++ b/src/tenable/client/index.ts @@ -8,6 +8,9 @@ import { AssetsExportStatusResponse, AssetsResponse, CancelExportResponse, + ComplianceChunk, + ComplianceExportStatusResponse, + ComplianceUuid, ContainerImageDetails, ContainerImagesResponse, ContainerRepositoryResponse, @@ -77,6 +80,35 @@ export default class TenableClient { return this.request('/users', Method.GET); } + public async exportComplianceData(options) { + return this.request( + '/compliance/export', + Method.POST, + options, + ); + } + + public async cancelComplianceFindingExport(exportUuid: string) { + return await this.request( + `/compliance/export/${exportUuid}/cancel`, + Method.POST, + ); + } + + public async fetchComplianceStatus(export_uuid) { + return this.request( + `/compliance/export/${export_uuid}/status`, + Method.GET, + ); + } + + public async fetchComplianceChunk(exportUuid: string, chunkId: number) { + return this.request( + `/compliance/export/${exportUuid}/chunks/${chunkId}`, + Method.GET, + ); + } + public async fetchScans() { return this.request('/scans', Method.GET); } diff --git a/src/tenable/client/types.ts b/src/tenable/client/types.ts index 405fd86..8f45a03 100644 --- a/src/tenable/client/types.ts +++ b/src/tenable/client/types.ts @@ -35,6 +35,82 @@ export interface User { uuid_id: string; } +export interface ComplianceUuid { + export_uuid: string; +} + +export interface ComplianceExportStatusResponse { + status: ExportStatus; + chunks_available: number[]; + chunks_failed: number[]; + chunks_cancelled: number[]; +} + +export interface VulnerabilitiesExportStatusResponse { + uuid: string; + status: ExportStatus; + chunks_available: number[]; + chunks_failed: number[]; + chunks_cancelled: number[]; + total_chunks: number; + chunks_available_count: number; + empty_chunks_count: number; + finished_chunks: number; + num_assets_per_chunk: number; + created: number; + filters?: ExportVulnerabilitiesFilter; +} + +export interface ComplianceChunk { + asset_uuid: string; + first_seen: string; + last_seen: string; + audit_file: string; + check_id: string; + check_name: string; + check_info: string; + expected_value: string; + actual_value: string; + status: string; + reference: { + framework: string; + control: string; + }[]; + see_also: string; + solution: string; + db_type: string; + plugin_id: number; + state: string; + description: string; + compliance_benchmark_name: string; + compliance_benchmark_version: string; + compliance_control_id: string; + compliance_full_id: string; + compliance_functional_id: string; + compliance_informational_id: string; + synopsis: string; + last_fixed: string; + last_observed: string; + metadata_id: string; + uname_output: string; + indexed_at: string; + plugin_name: string; + asset: complainceFindingExportAsset; +} + +export interface complainceFindingExportAsset { + id: string; + ipv4_addresses: string[]; + fqdns: string[]; + name: string; + agent_name: string; + agent_uuid: string; + netbios_name: string; + mac_addresses: string[]; + operating_systems: string[]; + system_type: string; +} + // -- https://cloud.tenable.com/scans // https://developer.tenable.com/reference#scans-list @@ -419,6 +495,11 @@ export interface UsersResponse { users: User[]; } +export interface ComplianceChunkResponse { + length: any; + complianceChunk: ComplianceChunk[]; +} + export interface ScansResponse { folders: Folder[]; scans: RecentScanSummary[]; @@ -586,6 +667,36 @@ export const VALID_VULNERABILITY_STATES = [ ] as const; export type VulnerabilityState = (typeof VALID_VULNERABILITY_STATES)[number]; +export const VALID_COMPLIANCE_STATES = ['OPEN', 'REOPENED', 'FIXED'] as const; +export type complianceChunkState = (typeof VALID_COMPLIANCE_STATES)[number]; + +export const VALID_COMPLIANCE_RESULT = [ + 'PASSED', + 'FAILED', + 'WARNING', + 'SKIPPED', + 'UNKNOWN', + 'ERROR', +] as const; +export type complianceChunkResult = (typeof VALID_COMPLIANCE_RESULT)[number]; + +export const VALID_COMPLIANCE_LAST_SEEN = [15, 30, 60, 90] as const; + +export type complianceChunkLastSeen = + (typeof VALID_COMPLIANCE_LAST_SEEN)[number]; + +export interface ExportComplianceFindingsFilter { + state?: complianceChunkState[]; + compliance_results?: complianceChunkResult[]; + last_seen?: number; +} + +export interface ExportComplianceFindingsOptions { + num_findings: number; + include_unlicensed?: boolean; + filters?: ExportComplianceFindingsFilter; +} + // Note: By default, vulnerability exports will only include // vulnerabilities found or fixed within the last 30 days if no // time-based filters (last_fixed, last_found, or first_found) are