From 36b344a4c58a3d78a892288e0eef71e9ff163b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 13 Dec 2024 15:31:03 -0300 Subject: [PATCH] [Telemetry][Security Solution] Index metadata collector (#194004) ## Summary Implements a security_solution task scheduled to run once a day to collect the following information: 1. Datastreams stats 2. Indices stats 3. ILMs stats 4. ILM configs The task allows a runtime configuration to limit the number of indices and data streams to analyze or event to disable the feature entirely. Once the data is gathered, the task sends it as EBT events. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Elastic Machine --- .../ftr_security_serverless_configs.yml | 1 + .buildkite/ftr_security_stateful_configs.yml | 1 + .../integration_tests/configuration.test.ts | 152 +++ .../server/integration_tests/lib/helpers.ts | 46 + .../lib/telemetry_helpers.ts | 26 +- .../integration_tests/telemetry.test.ts | 89 +- ...telemetry_detection_rules_preview_route.ts | 8 + .../utils/get_indices_metadata_preview.ts | 42 + .../telemetry/__mocks__/staging_indices.ts | 1160 +++++++++++++++++ .../server/lib/telemetry/async_sender.ts | 17 +- .../lib/telemetry/async_sender.types.ts | 5 +- .../lib/telemetry/collections_helpers.test.ts | 111 +- .../lib/telemetry/collections_helpers.ts | 150 +++ .../server/lib/telemetry/configuration.ts | 26 +- .../lib/telemetry/event_based/events.ts | 240 ++++ .../lib/telemetry/indices.metadata.types.ts | 68 + .../server/lib/telemetry/preview_sender.ts | 17 +- .../server/lib/telemetry/receiver.ts | 246 +++- .../server/lib/telemetry/sender.ts | 19 +- .../server/lib/telemetry/task.test.ts | 3 +- .../server/lib/telemetry/task.ts | 25 +- .../server/lib/telemetry/task_metrics.ts | 6 +- .../lib/telemetry/tasks/configuration.ts | 7 +- .../server/lib/telemetry/tasks/index.ts | 2 + .../lib/telemetry/tasks/indices.metadata.ts | 192 +++ .../server/lib/telemetry/types.ts | 9 + .../security_solution/server/plugin.ts | 3 +- .../detections_response/index.ts | 5 +- .../detections_response/tasks/index.ts | 9 + .../tasks/indices_metadata.ts | 108 ++ .../detections_response/tasks/task_manager.ts | 52 + .../check_registered_task_types.ts | 1 + .../config/ess/config.base.ts | 7 + .../config/ess/services.ts | 4 +- .../config/serverless/config.base.ts | 8 + .../config/serverless/services.ts | 2 + .../task_based/all_types.ts | 90 +- ...remove_time_fields_from_telemetry_stats.ts | 38 +- .../telemetry/configs/ess.config.ts | 23 + .../telemetry/configs/serverless.config.ts | 16 + .../test_suites/telemetry/index.ts | 13 + .../telemetry/tasks/indices_metadata.ts | 175 +++ .../tsconfig.json | 1 + 43 files changed, 3087 insertions(+), 136 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts create mode 100644 x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts create mode 100644 x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts create mode 100644 x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index 5ea5647d1d908..34280160fb7aa 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -89,6 +89,7 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml index 84a9c815bf565..a67bd2b299be8 100644 --- a/.buildkite/ftr_security_stateful_configs.yml +++ b/.buildkite/ftr_security_stateful_configs.yml @@ -70,6 +70,7 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts diff --git a/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts b/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts new file mode 100644 index 0000000000000..8629b54c7524d --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import Path from 'path'; +import axios from 'axios'; + +import { cloneDeep } from 'lodash'; + +import { telemetryConfiguration } from '../lib/telemetry/configuration'; +import { + TaskManagerPlugin, + type TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server/plugin'; + +import { + setupTestServers, + removeFile, + mockAxiosPost, + DEFAULT_GET_ROUTES, + mockAxiosGet, + getRandomInt, +} from './lib/helpers'; + +import { + type TestElasticsearchUtils, + type TestKibanaUtils, +} from '@kbn/core-test-helpers-kbn-server'; +import { Plugin as SecuritySolutionPlugin } from '../plugin'; +import { getTelemetryTasks, runSoonConfigTask } from './lib/telemetry_helpers'; +import type { SecurityTelemetryTask } from '../lib/telemetry/task'; + +jest.mock('axios'); + +const logFilePath = Path.join(__dirname, 'config.logs.log'); +const taskManagerStartSpy = jest.spyOn(TaskManagerPlugin.prototype, 'start'); +const securitySolutionStartSpy = jest.spyOn(SecuritySolutionPlugin.prototype, 'start'); + +const mockedAxiosGet = jest.spyOn(axios, 'get'); +const mockedAxiosPost = jest.spyOn(axios, 'post'); + +const securitySolutionPlugin = jest.spyOn(SecuritySolutionPlugin.prototype, 'start'); + +describe('configuration', () => { + let esServer: TestElasticsearchUtils; + let kibanaServer: TestKibanaUtils; + let taskManagerPlugin: TaskManagerStartContract; + let tasks: SecurityTelemetryTask[]; + + beforeAll(async () => { + await removeFile(logFilePath); + + const servers = await setupTestServers(logFilePath); + + esServer = servers.esServer; + kibanaServer = servers.kibanaServer; + + expect(taskManagerStartSpy).toHaveBeenCalledTimes(1); + taskManagerPlugin = taskManagerStartSpy.mock.results[0].value; + + expect(securitySolutionStartSpy).toHaveBeenCalledTimes(1); + + tasks = getTelemetryTasks(securitySolutionStartSpy); + + expect(securitySolutionPlugin).toHaveBeenCalledTimes(1); + }); + + afterAll(async () => { + if (kibanaServer) { + await kibanaServer.stop(); + } + if (esServer) { + await esServer.stop(); + } + }); + + beforeEach(async () => { + jest.clearAllMocks(); + mockAxiosPost(mockedAxiosPost); + }); + + afterEach(async () => {}); + + describe('configuration task', () => { + it('should keep default values when no new config was provided', async () => { + const before = cloneDeep(telemetryConfiguration); + + await runSoonConfigTask(tasks, taskManagerPlugin); + + expect(telemetryConfiguration).toEqual(before); + }); + + it('should update values with new manifest', async () => { + const expected = { + telemetry_max_buffer_size: getRandomInt(1, 100), + max_security_list_telemetry_batch: getRandomInt(1, 100), + max_endpoint_telemetry_batch: getRandomInt(1, 100), + max_detection_rule_telemetry_batch: getRandomInt(1, 100), + max_detection_alerts_batch: getRandomInt(1, 100), + use_async_sender: true, + pagination_config: { + max_page_size_bytes: getRandomInt(1, 100), + num_docs_to_sample: getRandomInt(1, 100), + }, + sender_channels: { + default: { + buffer_time_span_millis: getRandomInt(1, 100), + inflight_events_threshold: getRandomInt(1, 100), + max_payload_size_bytes: getRandomInt(1, 100), + }, + }, + indices_metadata_config: { + indices_threshold: getRandomInt(1, 100), + datastreams_threshold: getRandomInt(1, 100), + max_prefixes: getRandomInt(1, 100), + max_group_size: getRandomInt(1, 100), + }, + }; + + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [/.*telemetry-buffer-and-batch-sizes-v1.*/, { status: 200, data: cloneDeep(expected) }], + ]); + + await runSoonConfigTask(tasks, taskManagerPlugin); + + expect(telemetryConfiguration.telemetry_max_buffer_size).toEqual( + expected.telemetry_max_buffer_size + ); + expect(telemetryConfiguration.max_security_list_telemetry_batch).toEqual( + expected.max_security_list_telemetry_batch + ); + expect(telemetryConfiguration.max_endpoint_telemetry_batch).toEqual( + expected.max_endpoint_telemetry_batch + ); + expect(telemetryConfiguration.max_detection_rule_telemetry_batch).toEqual( + expected.max_detection_rule_telemetry_batch + ); + expect(telemetryConfiguration.max_detection_alerts_batch).toEqual( + expected.max_detection_alerts_batch + ); + expect(telemetryConfiguration.use_async_sender).toEqual(expected.use_async_sender); + expect(telemetryConfiguration.sender_channels).toEqual(expected.sender_channels); + expect(telemetryConfiguration.pagination_config).toEqual(expected.pagination_config); + expect(telemetryConfiguration.indices_metadata_config).toEqual( + expected.indices_metadata_config + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts index ccc73435636e9..d6eb3b6d8ca71 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts @@ -10,6 +10,20 @@ import Util from 'util'; import type { ElasticsearchClient } from '@kbn/core/server'; import deepmerge from 'deepmerge'; import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server'; + +export const DEFAULT_GET_ROUTES: Array<[RegExp, unknown]> = [ + [new RegExp('.*/ping$'), { status: 200 }], + [ + /.*kibana\/manifest\/artifacts.*/, + { + status: 200, + data: 'x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/kibana-artifacts.zip', + }, + ], +]; + +export const DEFAULT_POST_ROUTES: Array<[RegExp, unknown]> = [[/.*/, { status: 200 }]]; + const asyncUnlink = Util.promisify(Fs.unlink); /** @@ -127,3 +141,35 @@ export function updateTimestamps(data: object[]): object[] { return { ...d, '@timestamp': new Date(currentTimeMillis + (i + 1) * 100) }; }); } + +export function mockAxiosPost( + postSpy: jest.SpyInstance, + routes: Array<[RegExp, unknown]> = DEFAULT_POST_ROUTES +) { + postSpy.mockImplementation(async (url: string) => { + for (const [route, returnValue] of routes) { + if (route.test(url)) { + return returnValue; + } + } + return { status: 404 }; + }); +} + +export function mockAxiosGet( + getSpy: jest.SpyInstance, + routes: Array<[RegExp, unknown]> = DEFAULT_GET_ROUTES +) { + getSpy.mockImplementation(async (url: string) => { + for (const [route, returnValue] of routes) { + if (route.test(url)) { + return returnValue; + } + } + return { status: 404 }; + }); +} + +export function getRandomInt(min: number, max: number): number { + return Math.floor(Math.random() * (max - min + 1)) + min; +} diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index a36d5e1be38de..8dc9f1cbbca49 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -25,13 +25,13 @@ import { deleteExceptionListItem, } from '@kbn/lists-plugin/server/services/exception_lists'; import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants'; +import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import { packagePolicyService } from '@kbn/fleet-plugin/server/services'; import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants'; import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock'; -import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; -import { bulkInsert, updateTimestamps } from './helpers'; +import { bulkInsert, eventually, updateTimestamps } from './helpers'; import { TelemetryEventsSender } from '../../lib/telemetry/sender'; import type { SecuritySolutionPluginStart, @@ -41,6 +41,7 @@ import type { SecurityTelemetryTask } from '../../lib/telemetry/task'; import { Plugin as SecuritySolutionPlugin } from '../../plugin'; import { AsyncTelemetryEventsSender } from '../../lib/telemetry/async_sender'; import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/receiver'; +import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import mockEndpointAlert from '../__mocks__/endpoint-alert.json'; import mockedRule from '../__mocks__/rule.json'; import fleetAgents from '../__mocks__/fleet-agents.json'; @@ -417,3 +418,24 @@ export function getTelemetryTaskType(task: SecurityTelemetryTask): string { return ''; } } + +export async function runSoonConfigTask( + tasks: SecurityTelemetryTask[], + taskManagerPlugin: TaskManagerStartContract +) { + const configTaskType = 'security:telemetry-configuration'; + const configTask = getTelemetryTask(tasks, configTaskType); + const runAfter = new Date(); + await eventually(async () => { + await taskManagerPlugin.runSoon(configTask.getTaskId()); + }); + + // wait until the task finishes + await eventually(async () => { + const hasRun = await taskManagerPlugin + .get(configTask.getTaskId()) + .then((t) => new Date(t.state.lastExecutionTimestamp) > runAfter) + .catch(() => false); + expect(hasRun).toBe(true); + }); +} diff --git a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts index 801926adbf948..409eb867e833b 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts @@ -20,7 +20,14 @@ import { TELEMETRY_CHANNEL_ENDPOINT_META, } from '../lib/telemetry/constants'; -import { eventually, setupTestServers, removeFile } from './lib/helpers'; +import { + eventually, + setupTestServers, + removeFile, + mockAxiosGet, + mockAxiosPost, + DEFAULT_GET_ROUTES, +} from './lib/helpers'; import { cleanupMockedAlerts, cleanupMockedExceptionLists, @@ -37,6 +44,7 @@ import { mockEndpointData, getTelemetryReceiver, mockPrebuiltRulesData, + runSoonConfigTask, } from './lib/telemetry_helpers'; import { @@ -99,6 +107,7 @@ describe('telemetry tasks', () => { }, }, }); + esServer = servers.esServer; kibanaServer = servers.kibanaServer; @@ -134,7 +143,17 @@ describe('telemetry tasks', () => { beforeEach(async () => { jest.clearAllMocks(); - mockAxiosGet(); + mockAxiosPost(mockedAxiosPost); + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [ + /.*telemetry-buffer-and-batch-sizes-v1.*/, + { + status: 200, + data: fakeBufferAndSizesConfigAsyncDisabled, + }, + ], + ]); deferred = []; }); @@ -208,21 +227,17 @@ describe('telemetry tasks', () => { }); it('should use new sender when configured', async () => { - const configTaskType = 'security:telemetry-configuration'; - const configTask = getTelemetryTask(tasks, configTaskType); + mockAxiosPost(mockedAxiosPost); - mockAxiosGet(fakeBufferAndSizesConfigAsyncEnabled); - await eventually(async () => { - await taskManagerPlugin.runSoon(configTask.getTaskId()); - }); + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [ + /.*telemetry-buffer-and-batch-sizes-v1.*/, + { status: 200, data: fakeBufferAndSizesConfigAsyncEnabled }, + ], + ]); - // wait until the task finishes - await eventually(async () => { - const found = (await taskManagerPlugin.fetch()).docs.find( - (t) => t.taskType === configTaskType - ); - expect(found).toBeFalsy(); - }); + await runSoonConfigTask(tasks, taskManagerPlugin); const [task, started] = await mockAndScheduleDetectionRulesTask(); @@ -238,13 +253,20 @@ describe('telemetry tasks', () => { it('should update sender queue config', async () => { const expectedConfig = fakeBufferAndSizesConfigWithQueues.sender_channels['task-metrics']; - const configTaskType = 'security:telemetry-configuration'; - const configTask = getTelemetryTask(tasks, configTaskType); - mockAxiosGet(fakeBufferAndSizesConfigWithQueues); - await eventually(async () => { - await taskManagerPlugin.runSoon(configTask.getTaskId()); - }); + mockAxiosPost(mockedAxiosPost); + mockAxiosGet(mockedAxiosGet, [ + ...DEFAULT_GET_ROUTES, + [ + /.*telemetry-buffer-and-batch-sizes-v1.*/, + { + status: 200, + data: fakeBufferAndSizesConfigWithQueues, + }, + ], + ]); + + await runSoonConfigTask(tasks, taskManagerPlugin); await eventually(async () => { /* eslint-disable dot-notation */ @@ -838,31 +860,6 @@ describe('telemetry tasks', () => { }); } - function mockAxiosGet(bufferConfig: unknown = fakeBufferAndSizesConfigAsyncDisabled) { - mockedAxiosPost.mockImplementation( - async (_url: string, _data?: unknown, _config?: AxiosRequestConfig | undefined) => { - return { status: 200 }; - } - ); - - mockedAxiosGet.mockImplementation(async (url: string) => { - if (url.startsWith(ENDPOINT_STAGING) && url.endsWith('ping')) { - return { status: 200 }; - } else if (url.indexOf('kibana/manifest/artifacts') !== -1) { - return { - status: 200, - data: 'x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/kibana-artifacts.zip', - }; - } else if (url.indexOf('telemetry-buffer-and-batch-sizes-v1') !== -1) { - return { - status: 200, - data: bufferConfig, - }; - } - return { status: 404 }; - }); - } - async function getTaskMetricsRequests( task: SecurityTelemetryTask, olderThan: number diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts index 8013b2af9742b..4fd990e7f24fb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts @@ -15,6 +15,7 @@ import { getDetectionRulesPreview } from './utils/get_detecton_rules_preview'; import { getSecurityListsPreview } from './utils/get_security_lists_preview'; import { getEndpointPreview } from './utils/get_endpoint_preview'; import { getDiagnosticsPreview } from './utils/get_diagnostics_preview'; +import { getIndicesMetadataPreview } from './utils/get_indices_metadata_preview'; export const telemetryDetectionRulesPreviewRoute = ( router: SecuritySolutionPluginRouter, @@ -62,12 +63,19 @@ export const telemetryDetectionRulesPreviewRoute = ( telemetrySender, }); + const indicesMetadata = await getIndicesMetadataPreview({ + logger, + telemetryReceiver, + telemetrySender, + }); + return response.ok({ body: { detection_rules: detectionRules, security_lists: securityLists, endpoints, diagnostics, + indices_metadata: indicesMetadata, }, }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts new file mode 100644 index 0000000000000..7144034f91e71 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_indices_metadata_preview.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; + +import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender'; +import type { ITelemetryReceiver } from '../../../../telemetry/receiver'; +import { PreviewTaskMetricsService } from '../../../../telemetry/preview_task_metrics'; +import type { ITelemetryEventsSender } from '../../../../telemetry/sender'; +import { createTelemetryIndicesMetadataTaskConfig } from '../../../../telemetry/tasks/indices.metadata'; + +export const getIndicesMetadataPreview = async ({ + logger, + telemetryReceiver, + telemetrySender, +}: { + logger: Logger; + telemetryReceiver: ITelemetryReceiver; + telemetrySender: ITelemetryEventsSender; +}): Promise> => { + const taskExecutionPeriod = { + last: new Date(0).toISOString(), + current: new Date().toISOString(), + }; + + const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender); + const taskMetricsService = new PreviewTaskMetricsService(logger, taskSender); + const task = createTelemetryIndicesMetadataTaskConfig(); + await task.runTask( + 'indices-metadata-telemetry', + logger, + telemetryReceiver, + taskSender, + taskMetricsService, + taskExecutionPeriod + ); + return taskSender.getEbtEventsSent(); +}; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts new file mode 100644 index 0000000000000..e7fe78336ad06 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/staging_indices.ts @@ -0,0 +1,1160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const stagingIndices = [ + 'apm-7.14.2-profile-000011', + 'apm-7.14.2-profile-000012', + 'apm-7.14.2-profile-000013', + '.ds-logs-elastic_agent.filebeat-default-2023.07.22-000002', + 'apm-7.16.2-error-000010', + 'apm-7.16.2-error-000011', + '.ds-security_lists_telemetry_elastic-2024.04.15-000052', + 'security_solution_signals_test', + '.ds-v2_enrichment_data_cloud-2023.09.14-000012', + '.internal.alerts-transform.health.alerts-default-000001', + '.ds-security_lists_telemetry_elastic-2024.03.16-000050', + 'apm-7.16.2-span-000001', + 'apm-7.16.2-span-000002', + 'apm-7.16.2-span-000003', + 'apm-7.16.2-span-000004', + 'apm-7.16.2-span-000005', + 'apm-7.16.2-span-000006', + 'apm-7.16.2-span-000007', + 'apm-7.16.2-span-000008', + 'apm-7.16.2-span-000009', + '.ds-metrics-elastic_agent.filebeat-default-2023.08.21-000003', + '.ds-detonate_results_telemetry-2022.08.18-000009', + 'apm-7.14.2-span-000001', + 'apm-7.14.2-span-000002', + 'apm-7.14.2-span-000003', + '.ds-task_metrics_elastic-2023.10.18-000010', + 'apm-7.14.2-span-000004', + 'apm-7.14.2-span-000006', + 'apm-7.14.2-span-000005', + 'apm-7.14.2-span-000008', + 'apm-7.14.2-span-000007', + 'apm-7.14.2-span-000009', + 'apm-7.16.2-span-000010', + 'apm-7.16.2-span-000011', + 'shrink-0gwu-.ds-alert_telemetry_elastic-2023.10.31-000041', + '.ds-security_lists_telemetry_elastic-2023.11.17-000042', + 'shrink-y3pl-.ds-alert_telemetry_elastic-2023.11.30-000044', + '.ds-metrics-elastic_agent.filebeat-default-2023.07.22-000002', + '.ds-detonate_results_telemetry-2022.07.19-000008', + 'apm-7.16.1-profile-000001', + 'apm-7.16.1-profile-000002', + 'apm-7.16.1-profile-000003', + 'apm-7.16.1-profile-000004', + 'apm-7.16.1-profile-000005', + 'apm-7.16.1-profile-000006', + 'apm-7.16.1-profile-000007', + 'apm-7.16.1-profile-000008', + 'apm-7.16.1-profile-000009', + 'apm-7.14.2-span-000011', + '.ds-metrics-apm.app.apm_server-default-2024.06.24-000012', + 'apm-7.14.2-span-000010', + 'apm-7.14.2-span-000013', + 'apm-7.14.2-span-000012', + 'apm-7.14.2-error-000001', + 'apm-7.14.2-error-000002', + 'apm-7.14.2-error-000003', + 'apm-7.14.2-error-000004', + 'apm-7.14.2-error-000005', + 'apm-7.14.2-error-000006', + 'apm-7.14.2-error-000007', + 'apm-7.14.2-error-000008', + 'apm-7.14.2-error-000009', + 'shrink-kbvc-.ds-kibana-snapshot-2024.04.24-000046', + '.kibana-observability-ai-assistant-conversations-000001', + 'apm-7.16.1-profile-000010', + '.kibana_7.12.0_001', + 'apm-7.16.1-profile-000011', + 'apm-7.14.2-error-000010', + 'apm-7.14.2-error-000011', + 'apm-7.14.2-error-000012', + 'apm-7.14.2-error-000013', + '.ds-yaas_alert_telemetry-2022.06.11-000009', + '.ds-cluster_telemetry_elastic-2024.06.05-000058', + '.ds-detections_alert_telemetry_elastic-2024.09.29-000079', + 'apm-7.13.2-profile-000001', + 'apm-7.13.2-profile-000002', + 'apm-7.13.2-profile-000003', + 'apm-7.13.2-profile-000004', + 'apm-7.13.2-profile-000005', + 'apm-7.13.2-profile-000006', + 'apm-7.13.2-profile-000007', + 'apm-7.13.2-profile-000008', + 'apm-7.13.2-profile-000009', + 'model_tracking_latest-1', + 'model_tracking_latest-2', + 'model_tracking_latest-3', + 'model_tracking_latest-6', + '.ds-yaas_alert_telemetry-2022.05.12-000008', + '.ds-cluster_telemetry_elastic-2024.05.06-000056', + 'model_tracking_latest-8', + 'ia-cti-triaged-alerts', + 'apm-7.13.2-profile-000010', + 'apm-7.13.2-profile-000011', + 'apm-7.13.2-profile-000012', + 'apm-7.13.2-profile-000013', + 'apm-7.13.2-profile-000014', + 'apm-7.13.2-profile-000015', + 'apm-7.13.2-profile-000016', + 'apm-7.13.2-profile-000017', + '.ds-alert_timelines_elastic-2024.04.28-000050', + 'shrink-iriq-.ds-kibana-snapshot-2024.03.25-000043', + '.ds-detection_rule_cluster_details-2022.11.05-000010', + 'apm-7.15.1-transaction-000004', + 'apm-7.10.0-transaction-000001', + 'apm-7.10.0-transaction-000002', + 'apm-7.10.0-transaction-000003', + 'apm-7.10.0-transaction-000004', + 'apm-7.10.0-transaction-000005', + 'apm-7.10.0-transaction-000006', + 'apm-7.10.0-transaction-000007', + 'apm-7.10.0-transaction-000008', + 'apm-7.10.0-transaction-000009', + '.ds-metrics-fleet_server.agent_status-default-2024.07.20-000002', + '.ds-yaas_alert_telemetry-2022.09.09-000012', + '.ds-alert_timelines_elastic-2024.03.29-000047', + 'apm-7.15.1-transaction-000009', + 'apm-7.15.1-transaction-000008', + 'apm-7.15.1-transaction-000007', + 'apm-7.15.1-transaction-000006', + 'apm-7.15.1-transaction-000005', + 'apm-7.15.1-transaction-000003', + 'apm-7.15.1-transaction-000002', + 'model_tracking_latest-a', + 'apm-7.15.1-transaction-000001', + 'model_tracking_latest-d', + 'model_tracking_latest-e', + 'test_metadata', + '.ds-endpoint_metadata_telemetry_elastic-2024.02.18-000055', + 'apm-7.15.1-transaction-000011', + 'apm-7.15.1-transaction-000012', + 'apm-7.15.1-transaction-000013', + 'apm-7.10.0-transaction-000010', + 'apm-7.10.0-transaction-000011', + 'apm-7.10.0-transaction-000012', + 'apm-7.10.0-transaction-000013', + 'apm-7.10.0-transaction-000014', + 'apm-7.10.0-transaction-000015', + 'apm-7.10.0-transaction-000016', + 'apm-7.10.0-transaction-000017', + 'apm-7.10.0-transaction-000018', + 'apm-7.10.0-transaction-000019', + 'apm-7.15.1-transaction-000010', + '.ds-endpoint_metadata_telemetry_elastic-2024.01.19-000053', + 'apm-7.10.0-transaction-000020', + 'apm-7.10.0-transaction-000021', + 'apm-7.10.0-transaction-000022', + 'apm-7.10.0-transaction-000023', + 'apm-7.10.0-transaction-000024', + '.ds-alert_telemetry_elastic-2024.05.28-000069', + '.ds-alert_telemetry_elastic-2024.06.27-000072', + 'apm-7.16.1-error-000001', + 'apm-7.16.1-error-000002', + 'apm-7.16.1-error-000003', + 'apm-7.16.1-error-000004', + 'apm-7.16.1-error-000005', + 'apm-7.16.1-error-000006', + 'apm-7.16.1-error-000007', + 'apm-7.16.1-error-000008', + 'apm-7.16.1-error-000009', + '.ds-endpoint_metadata_telemetry_elastic-2024.09.15-000072', + 'apm-7.15.1-profile-000001', + 'apm-7.15.1-profile-000002', + 'apm-7.15.1-profile-000003', + 'apm-7.15.1-profile-000004', + 'apm-7.15.1-profile-000005', + 'apm-7.15.1-profile-000006', + 'apm-7.15.1-profile-000007', + 'apm-7.15.1-profile-000008', + 'apm-7.15.1-profile-000009', + 'shrink-ujrr-.ds-detections_alert_telemetry_elastic-2024.01.03-000046', + 'apm-7.16.1-error-000010', + 'apm-7.16.1-error-000011', + '.internal.alerts-ml.anomaly-detection.alerts-default-000001', + '.ds-endpoint_metadata_telemetry_elastic-2024.08.16-000070', + '.ds-security_lists_telemetry_elastic-2024.02.15-000048', + '.ds-endpoint_metadata_telemetry_elastic-2024.07.17-000068', + '.ds-v2_enrichment_data_cloud-2023.10.14-000013', + 'apm-7.15.1-profile-000010', + 'apm-7.15.1-profile-000011', + 'apm-7.15.1-profile-000012', + 'apm-7.15.1-profile-000013', + 'us-airports', + '.ds-insights_telemetry_elastic-2024.03.10-000042', + 'apm-7.13.2-onboarding-2021.06.15', + '.ds-security_lists_telemetry_elastic-2024.01.16-000046', + '.ds-detonate_results_telemetry-2022.10.17-000011', + '.ds-security_lists_telemetry_elastic-2024.09.12-000062', + 'test-index-1', + 'filebeat-7.12.0-2021.04.08', + '.siem-signals-default-000001', + '.siem-signals-default-000002', + '.siem-signals-default-000003', + '.siem-signals-default-000004', + '.siem-signals-default-000005', + '.siem-signals-default-000006', + '.siem-signals-default-000007', + '.siem-signals-default-000008', + '.siem-signals-default-000009', + 'shrink-b_x8-.ds-detections_alert_telemetry_elastic-2023.10.05-000037', + '.ds-security_lists_telemetry_elastic-2024.08.13-000060', + '.ds-security_lists_telemetry_elastic-2024.07.14-000058', + '.siem-signals-default-000010', + '.siem-signals-default-000011', + '.siem-signals-default-000012', + '.siem-signals-default-000013', + '.siem-signals-default-000014', + '.siem-signals-default-000015', + '.siem-signals-default-000016', + '.siem-signals-default-000017', + '.ds-sample-tags-tracking-2022.06.08-000006', + '.siem-signals-default-000018', + '.siem-signals-default-000019', + '.ds-inline_lookup_decision_tracking-2022.06.05-000006', + '.slm-history-3-000004', + '.kibana_task_manager_7.17.0_001', + 'apm-7.13.2-span-000001', + 'apm-7.13.2-span-000002', + 'apm-7.13.2-span-000003', + 'apm-7.13.2-span-000004', + 'apm-7.13.2-span-000005', + 'apm-7.13.2-span-000006', + '.siem-signals-default-000020', + '.siem-signals-default-000021', + '.siem-signals-default-000022', + '.siem-signals-default-000023', + 'apm-7.13.2-span-000008', + 'apm-7.13.2-span-000009', + 'apm-7.13.2-span-000007', + '.ds-inline_lookup_decision_tracking-2022.05.06-000005', + '.ds-sample-tags-tracking-2022.05.09-000005', + 'latest-cluster_telemetry_elastic', + '.ds-detection_rule_cluster_metrics-2022.11.05-000010', + 'apm-7.11.0-error-000001', + 'apm-7.11.0-error-000002', + 'apm-7.11.0-error-000003', + '.ds-logs-elastic_agent.metricbeat-default-2023.08.21-000003', + 'apm-7.11.0-error-000005', + 'apm-7.11.0-error-000006', + 'apm-7.11.0-error-000007', + 'apm-7.11.0-error-000008', + 'apm-7.11.0-transaction-000001', + '.ds-metrics-fleet_server.agent_status-default-2024.09.18-000004', + 'apm-7.11.0-transaction-000003', + 'apm-7.11.0-transaction-000002', + 'apm-7.11.0-transaction-000005', + 'apm-7.11.0-transaction-000006', + 'apm-7.11.0-transaction-000007', + 'apm-7.11.0-transaction-000008', + '.ds-cluster_telemetry_elastic-2023.10.09-000042', + 'apm-7.11.0-transaction-000009', + 'apm-7.11.0-transaction-000004', + 'apm-7.11.0-error-000009', + 'apm-7.11.0-error-000004', + 'apm-7.12.0-profile-000008', + 'apm-7.12.0-profile-000009', + 'apm-7.12.0-profile-000001', + 'apm-7.12.0-profile-000002', + 'apm-7.12.0-profile-000003', + 'apm-7.12.0-profile-000004', + 'apm-7.12.0-profile-000005', + 'apm-7.12.0-profile-000006', + 'apm-7.12.0-profile-000007', + 'apm-7.11.0-error-000010', + '.kibana_7.16.1_001', + 'apm-7.11.0-error-000012', + 'apm-7.11.0-error-000011', + 'apm-7.11.0-error-000014', + 'apm-7.11.0-error-000015', + '.ds-metrics-fleet_server.agent_status-default-2024.08.19-000003', + 'rollup-cluster_telemetry_elastic', + 'apm-7.11.0-error-000018', + 'apm-7.11.0-error-000019', + 'apm-7.11.0-error-000017', + 'apm-7.11.0-error-000016', + 'apm-7.11.0-error-000013', + 'apm-7.12.0-profile-000010', + 'apm-7.12.0-profile-000011', + '.ds-telemetry_usage-2024.06.01-000046', + 'apm-7.12.0-profile-000012', + 'apm-7.12.0-profile-000013', + 'apm-7.12.0-profile-000014', + 'apm-7.12.0-profile-000015', + 'apm-7.12.0-profile-000016', + 'apm-7.12.0-profile-000019', + 'apm-7.12.0-profile-000017', + 'apm-7.12.0-profile-000018', + 'apm-7.13.2-span-000015', + 'apm-7.13.2-span-000016', + 'apm-7.13.2-span-000013', + 'shrink-h4sx-.ds-detections_alert_telemetry_elastic-2024.03.03-000052', + 'apm-7.13.2-span-000014', + 'apm-7.13.2-span-000011', + 'apm-7.13.2-span-000012', + 'apm-7.11.0-error-000020', + 'apm-7.11.0-error-000021', + 'apm-7.11.0-error-000022', + 'apm-7.13.2-span-000010', + 'apm-7.13.2-span-000017', + 'apm-7.11.0-transaction-000014', + 'apm-7.11.0-transaction-000013', + 'apm-7.11.0-transaction-000012', + 'apm-7.11.0-transaction-000011', + 'apm-7.11.0-transaction-000010', + 'apm-7.11.0-transaction-000019', + 'apm-7.11.0-transaction-000018', + 'apm-7.12.0-profile-000020', + 'apm-7.11.0-transaction-000017', + 'apm-7.11.0-transaction-000016', + '.ds-alert_timelines_elastic-2024.09.25-000066', + 'apm-7.11.0-transaction-000015', + 'apm-7.11.0-transaction-000022', + 'apm-7.11.0-transaction-000021', + 'apm-7.11.0-transaction-000020', + 'shrink-wl9p-.ds-alert_telemetry_elastic-2024.02.28-000053', + '.ds-logs-elastic_agent.metricbeat-default-2023.07.22-000002', + '.ds-telemetry_usage-2023.09.05-000028', + 'apm-7.12.0-metric-000001', + 'apm-7.12.0-metric-000002', + 'apm-7.12.0-metric-000003', + 'apm-7.12.0-metric-000004', + 'apm-7.12.0-metric-000005', + 'apm-7.12.0-metric-000006', + 'apm-7.12.0-metric-000007', + 'apm-7.12.0-metric-000008', + 'apm-7.12.0-metric-000009', + '.ds-alert_timelines_elastic-2024.08.26-000063', + '.ds-logs-elastic_agent-default-2023.08.21-000003', + 'apm-7.17.0-span-000001', + 'apm-7.17.0-span-000002', + 'apm-7.17.0-span-000003', + 'apm-7.17.0-span-000004', + 'apm-7.17.0-span-000005', + 'apm-7.17.0-span-000006', + '.lists-default-000001', + 'apm-7.17.0-span-000008', + 'apm-7.17.0-span-000009', + 'apm-7.17.0-span-000007', + '.ds-download_stats_telemetry_elastic-2023.03.09-000018', + 'apm-7.12.0-metric-000010', + 'apm-7.12.0-metric-000011', + 'apm-7.12.0-metric-000012', + 'apm-7.12.0-metric-000013', + 'apm-7.12.0-metric-000014', + 'apm-7.12.0-metric-000015', + 'apm-7.12.0-metric-000016', + 'apm-7.12.0-metric-000017', + 'apm-7.12.0-metric-000018', + 'apm-7.12.0-metric-000019', + '.ds-alert_timelines_elastic-2024.07.27-000060', + 'detection_alert_telemetry_elastic', + '.kibana_7.14.2_001', + '.ds-logs-elastic_agent-default-2023.07.22-000002', + 'failed-docs-000001', + 'apm-7.12.0-metric-000020', + '.ds-insights_telemetry_elastic-2024.01.10-000038', + 'metrics-endpoint.metadata_current_default', + '.ds-detections_alert_telemetry_elastic-2024.07.01-000067', + 'apm-7.16.2-transaction-000001', + 'apm-7.16.2-transaction-000002', + 'apm-7.16.2-transaction-000003', + 'apm-7.16.2-transaction-000004', + 'apm-7.16.2-transaction-000005', + 'apm-7.16.2-transaction-000006', + 'apm-7.16.2-transaction-000007', + 'apm-7.16.2-transaction-000008', + 'apm-7.16.2-transaction-000009', + 'apm-7.10.2-metric-000001', + 'apm-7.10.2-metric-000002', + 'apm-7.10.2-metric-000003', + 'apm-7.10.2-metric-000004', + 'apm-7.10.2-metric-000005', + 'apm-7.10.2-metric-000006', + 'apm-7.10.2-metric-000007', + 'apm-7.10.2-metric-000008', + 'apm-7.10.2-metric-000009', + '.kibana_task_manager_7.12.0_001', + '.ds-logs-generic-default-2023.09.07-000002', + 'apm-7.13.2-transaction-000001', + 'apm-7.13.2-transaction-000002', + 'apm-7.13.2-transaction-000003', + 'apm-7.13.2-transaction-000004', + 'apm-7.13.2-transaction-000005', + 'apm-7.13.2-transaction-000006', + 'apm-7.13.2-transaction-000007', + 'apm-7.13.2-transaction-000008', + 'apm-7.13.2-transaction-000009', + 'apm-7.16.2-transaction-000011', + 'apm-7.16.2-transaction-000010', + 'apm-7.10.0-metric-000004', + 'apm-7.10.0-metric-000003', + 'apm-7.10.0-metric-000002', + 'apm-7.10.0-metric-000001', + 'apm-7.10.2-metric-000010', + 'apm-7.10.2-metric-000011', + 'apm-7.10.2-metric-000012', + 'apm-7.10.2-metric-000013', + 'apm-7.10.2-metric-000014', + 'apm-7.10.2-metric-000015', + 'apm-7.10.2-metric-000016', + 'apm-7.10.2-metric-000017', + 'apm-7.10.2-metric-000018', + 'apm-7.10.2-metric-000019', + 'apm-7.10.0-metric-000008', + 'apm-7.10.0-metric-000007', + 'apm-7.10.0-metric-000006', + 'shrink-_z0f-.ds-alert_timelines_elastic-2024.02.28-000044', + '.ds-security_lists_telemetry_elastic-2023.12.17-000044', + 'apm-7.13.2-transaction-000010', + 'apm-7.13.2-transaction-000011', + 'apm-7.13.2-transaction-000012', + 'apm-7.13.2-transaction-000013', + 'apm-7.13.2-transaction-000014', + 'apm-7.13.2-transaction-000015', + 'apm-7.13.2-transaction-000016', + 'apm-7.13.2-transaction-000017', + '.ds-metrics-fleet_server.agent_versions-default-2024.09.18-000004', + 'apm-7.10.0-metric-000005', + 'apm-7.10.0-metric-000011', + 'apm-7.10.0-metric-000010', + '.ds-kibana-snapshot-2024.09.21-000059', + 'apm-7.10.0-metric-000015', + 'apm-7.10.0-metric-000014', + 'apm-7.10.0-metric-000013', + 'apm-7.10.2-metric-000020', + 'apm-7.10.2-metric-000021', + 'apm-7.10.2-metric-000022', + 'apm-7.10.2-metric-000023', + 'apm-7.10.0-metric-000012', + 'apm-7.10.0-metric-000019', + 'test-rollup', + 'apm-7.10.0-metric-000018', + 'apm-7.10.0-metric-000017', + 'apm-7.10.0-metric-000016', + 'apm-7.10.0-metric-000009', + 'idx_processed_security-endpoint-metadata', + '.ds-logs-generic-default-2023.08.08-000001', + '.ds-metrics-fleet_server.agent_versions-default-2024.08.19-000003', + '.ds-detonate_results_telemetry-2022.05.20-000006', + 'apm-7.10.0-metric-000020', + 'apm-7.10.0-metric-000021', + 'apm-7.10.0-metric-000022', + 'apm-7.10.0-metric-000023', + 'apm-7.10.0-metric-000024', + '.ds-kibana-snapshot-2024.08.22-000057', + 'shrink-u5p4-.ds-kibana-snapshot-2024.05.24-000049', + '.ds-yaas_alert_telemetry-2022.02.11-000005', + '.ds-kibana-snapshot-2024.07.23-000054', + '.ds-detonate_results_telemetry-2022.09.17-000010', + '.kibana_7.13.2_001', + 'apm-7.15.1-metric-000001', + 'apm-7.15.1-metric-000002', + 'apm-7.15.1-metric-000003', + 'apm-7.15.1-metric-000004', + 'apm-7.15.1-metric-000005', + 'apm-7.15.1-metric-000006', + 'apm-7.15.1-metric-000007', + 'apm-7.15.1-metric-000008', + 'apm-7.15.1-metric-000009', + '.ds-logs-ingest-vt-reports-test-default-2023.06.23-000001', + '.ds-yaas_alert_telemetry-2022.01.12-000004', + '.internal.alerts-default.alerts-default-000001', + 'apm-7.15.1-metric-000010', + 'apm-7.15.1-metric-000011', + 'apm-7.15.1-metric-000012', + 'apm-7.15.1-metric-000013', + '.ds-telemetry_usage-2023.11.04-000032', + 'apm-7.10.2-span-000001', + 'apm-7.10.2-span-000002', + 'apm-7.10.2-span-000003', + 'apm-7.10.2-span-000004', + 'apm-7.10.2-span-000005', + 'apm-7.10.2-span-000006', + 'apm-7.10.2-span-000007', + 'apm-7.10.2-span-000008', + 'apm-7.10.2-span-000009', + '.ds-telemetry_usage-2023.10.05-000030', + 'apm-7.10.2-span-000010', + 'apm-7.10.2-span-000011', + 'apm-7.10.2-span-000012', + '.fleet-enrollment-api-keys-7', + 'apm-7.10.2-span-000013', + 'apm-7.10.2-span-000014', + 'apm-7.10.2-span-000016', + 'apm-7.10.2-span-000017', + 'apm-7.10.2-span-000018', + 'apm-7.10.2-span-000019', + 'apm-7.10.2-span-000015', + 'summary-cluster_telemetry_elastic', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.08.21-000003', + '.ds-detections_alert_telemetry_elastic_v2-2023.01.11-000001', + 'apm-7.10.2-span-000020', + '.ds-v2_alert_telemetry_elastic-2023.01.11-000001', + 'apm-7.10.2-span-000021', + 'apm-7.10.2-span-000023', + 'apm-7.10.2-span-000022', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.07.22-000002', + '.fleet-artifacts-7', + '.kibana_7.16.2_001', + '.ds-machine_learning_cluster_details-2022.11.07-000010', + '.ds-cluster_telemetry_elastic-2023.09.09-000040', + 'threatintel-2021.08.19-000001', + 'ilm-history-3-000004', + 'my-index-permissions', + '.ds-detections_alert_telemetry_elastic-2024.04.02-000058', + '.ds-alert_telemetry_endgame-2022.11.05-000086', + '.ds-endpoint_metadata_telemetry_elastic-2023.09.21-000045', + 'shrink-jwq5-.ds-alert_timelines_elastic-2024.01.29-000041', + '.ds-insights_telemetry_elastic-2024.09.06-000054', + 'kibana_snapshot', + '.ds-insights_telemetry_elastic-2023.12.11-000036', + 'metrics-index_pattern_placeholder', + '.ds-insights_telemetry_elastic-2024.08.07-000052', + 'shrink-48kl-.ds-detections_alert_telemetry_elastic-2023.12.04-000043', + 'apm-7.17.0-metric-000001', + 'failure-alert_telemetry_unified', + 'apm-7.17.0-metric-000002', + 'apm-7.17.0-metric-000003', + 'apm-7.17.0-metric-000004', + 'apm-7.17.0-metric-000005', + '.ds-insights_telemetry_elastic-2024.07.08-000050', + 'apm-7.17.0-metric-000006', + 'shrink-gqzj-.ds-detections_alert_telemetry_elastic-2024.02.02-000049', + 'apm-7.17.0-metric-000007', + 'apm-7.17.0-metric-000008', + 'apm-7.17.0-metric-000009', + '.ds-inline_lookup_decision_tracking-2022.02.05-000002', + '.ds-sample-tags-tracking-2021.12.21-000001', + '.kibana_task_manager_7.16.1_001', + '.ds-sample-tags-tracking-2022.02.08-000002', + '.ds-model_tracking_historical-2021.05.11-000004', + 'my-index-000001', + 'my-index-000002', + 'shrink-cgef-.ds-alert_telemetry_elastic-2024.01.29-000050', + '.ds-inline_lookup_decision_tracking-2022.01.06-000001', + '.security-tokens-7', + '.ds-telemetry_usage-2024.01.03-000036', + 'shrink-ixws-.ds-alert_timelines_elastic-2023.12.21-000037', + 'shrink-ud5m-.ds-alert_timelines_elastic-2023.12.30-000038', + '.ds-sample-tags-tracking-2022.03.10-000003', + 'apm-7.10.2-error-000001', + 'apm-7.10.2-error-000002', + 'apm-7.10.2-error-000003', + 'apm-7.10.2-error-000004', + 'apm-7.10.2-error-000005', + 'apm-7.10.2-error-000006', + 'apm-7.10.2-error-000007', + 'apm-7.10.2-error-000008', + 'apm-7.10.2-error-000009', + '.transform-notifications-000002', + '.ds-cluster_telemetry_elastic-2024.04.06-000054', + 'apm-7.17.0-profile-000001', + 'apm-7.17.0-profile-000002', + 'apm-7.17.0-profile-000003', + 'apm-7.17.0-profile-000004', + 'apm-7.17.0-profile-000005', + 'apm-7.17.0-profile-000006', + 'apm-7.17.0-profile-000007', + 'apm-7.17.0-profile-000008', + 'apm-7.17.0-profile-000009', + 'apm-7.10.2-error-000015', + 'apm-7.10.2-error-000014', + 'apm-7.10.2-error-000017', + 'apm-7.10.2-error-000016', + 'apm-7.10.2-error-000011', + 'apm-7.10.2-error-000010', + 'apm-7.10.2-error-000013', + 'apm-7.10.2-error-000012', + 'apm-7.10.2-error-000019', + 'apm-7.10.2-error-000018', + 'shrink-pcsa-.ds-alert_telemetry_elastic-2023.10.27-000040', + '.ds-yaas_alert_telemetry-2022.03.13-000006', + '.ds-metrics-fleet_server.agent_status-default-2024.06.20-000001', + 'shrink-d-hy-.ds-alert_telemetry_elastic-2023.12.30-000047', + '.ds-cluster_telemetry_elastic-2024.03.07-000052', + 'alert_telemetry_labels', + 'apm-7.10.2-error-000020', + 'apm-7.10.2-error-000021', + 'apm-7.10.2-error-000022', + 'apm-7.10.2-error-000023', + '.ds-yaas_alert_telemetry-2021.12.13-000003', + '.security-7', + '.ds-cluster_telemetry_elastic-2023.11.08-000044', + 'apm-7.16.2-profile-000001', + 'apm-7.16.2-profile-000002', + 'apm-7.16.2-profile-000003', + 'apm-7.16.2-profile-000004', + 'apm-7.16.2-profile-000005', + 'apm-7.16.2-profile-000006', + 'apm-7.16.2-profile-000007', + 'apm-7.16.2-profile-000008', + 'apm-7.16.2-profile-000009', + 'apm-7.16.1-transaction-000001', + 'apm-7.16.1-transaction-000003', + 'apm-7.16.1-transaction-000006', + 'apm-7.16.1-transaction-000007', + 'apm-7.16.1-transaction-000008', + 'apm-7.16.1-transaction-000009', + 'apm-7.16.1-transaction-000002', + 'apm-7.16.1-transaction-000005', + 'apm-7.16.1-transaction-000004', + 'apm-7.10.2-onboarding-2021.02.03', + 'apm-7.10.2-onboarding-2021.02.04', + '.ds-alert_telemetry_elastic-2024.04.28-000066', + 'apm-7.16.2-profile-000010', + 'apm-7.16.2-profile-000011', + '.ds-logs-vt_reports_gold_table_export-default-2023.06.22-000001', + 'apm-7.16.1-transaction-000010', + 'apm-7.16.1-transaction-000011', + '.ds-endpoint_metadata_telemetry_elastic-2023.11.20-000049', + '.internal.alerts-ml.anomaly-detection-health.alerts-default-000001', + '.ds-enrichment_data_cloud-2022.12.02-000013', + '.ds-alert_telemetry_elastic-2024.03.29-000062', + '.kibana_task_manager_7.15.1_001', + '.ds-endpoint_metadata_telemetry_elastic-2023.10.21-000047', + '.ds-endpoint_metadata_telemetry_elastic-2024.06.17-000066', + '.internal.alerts-stack.alerts-default-000001', + 'apm-7.13.2-metric-000001', + 'apm-7.13.2-metric-000002', + 'apm-7.13.2-metric-000003', + 'apm-7.13.2-metric-000004', + 'apm-7.13.2-metric-000005', + 'apm-7.13.2-metric-000006', + 'apm-7.13.2-metric-000007', + 'apm-7.13.2-metric-000008', + 'apm-7.13.2-metric-000009', + '.enrich-enrich-alerts-cust-info-1618854141382', + '.enrich-enrich-alerts-hash-1618853111243', + '.ds-endpoint_metadata_telemetry_elastic-2024.05.18-000064', + 'apm-7.13.2-metric-000010', + 'apm-7.13.2-metric-000011', + 'apm-7.13.2-metric-000012', + 'apm-7.13.2-metric-000013', + 'apm-7.13.2-metric-000014', + 'apm-7.13.2-metric-000015', + 'apm-7.13.2-metric-000016', + 'apm-7.13.2-metric-000017', + 'security_solution_signals_test2', + 'security_solution_signals_test0', + 'security_solution_signals_test9', + '.internal.alerts-observability.apm.alerts-default-000001', + '.kibana_task_manager_7.13.2_001', + 'apm-7.13.2-error-000001', + 'apm-7.13.2-error-000002', + 'apm-7.13.2-error-000003', + 'apm-7.13.2-error-000004', + 'apm-7.13.2-error-000005', + 'apm-7.13.2-error-000006', + 'apm-7.13.2-error-000007', + 'apm-7.13.2-error-000008', + 'apm-7.13.2-error-000009', + '.ds-insights_telemetry_elastic-2024.04.09-000044', + '.ds-security_lists_telemetry_elastic-2024.06.14-000056', + 'apm-7.11.0-span-000001', + 'apm-7.11.0-span-000002', + 'apm-7.11.0-span-000003', + 'apm-7.11.0-span-000004', + 'apm-7.11.0-span-000005', + 'apm-7.11.0-span-000006', + 'apm-7.11.0-span-000007', + 'apm-7.11.0-span-000008', + 'apm-7.11.0-span-000009', + '.internal.alerts-observability.uptime.alerts-default-000001', + 'apm-7.13.2-error-000010', + 'apm-7.13.2-error-000011', + 'apm-7.13.2-error-000012', + 'apm-7.13.2-error-000013', + 'apm-7.13.2-error-000014', + 'apm-7.13.2-error-000015', + 'apm-7.13.2-error-000016', + 'apm-7.13.2-error-000017', + 'apm-7.10.2-onboarding-2021.01.14', + '.fleet-servers-7', + '.ds-security_lists_telemetry_elastic-2024.05.15-000054', + 'apm-7.11.0-span-000010', + 'apm-7.11.0-profile-000001', + 'apm-7.11.0-profile-000002', + 'apm-7.11.0-profile-000003', + 'apm-7.11.0-profile-000004', + 'apm-7.11.0-profile-000005', + 'apm-7.11.0-profile-000006', + 'apm-7.11.0-profile-000007', + 'apm-7.11.0-profile-000008', + 'apm-7.11.0-profile-000009', + 'apm-7.11.0-span-000014', + 'apm-7.11.0-span-000013', + 'apm-7.11.0-span-000016', + 'apm-7.11.0-span-000015', + 'apm-7.11.0-span-000018', + 'apm-7.11.0-span-000017', + 'apm-7.11.0-span-000019', + 'apm-7.11.0-span-000012', + 'apm-7.11.0-span-000011', + '.fleet-policies-leader-7', + '.ds-metrics-fleet_server.agent_versions-default-2024.06.20-000001', + '.ds-sample-tags-tracking-2022.04.09-000004', + 'security-rollups-hourly', + 'test-bq-export', + '.internal.alerts-observability.slo.alerts-default-000001', + 'apm-7.11.0-profile-000010', + 'apm-7.11.0-profile-000011', + 'apm-7.11.0-profile-000012', + 'apm-7.11.0-profile-000013', + 'apm-7.11.0-profile-000014', + 'apm-7.11.0-profile-000015', + 'apm-7.11.0-profile-000016', + 'apm-7.11.0-profile-000017', + 'apm-7.11.0-profile-000018', + 'apm-7.11.0-profile-000019', + 'apm-7.11.0-span-000021', + 'apm-7.11.0-span-000020', + '.ds-inline_lookup_decision_tracking-2022.03.07-000003', + 'apm-7.11.0-span-000022', + 'apm-7.10.2-transaction-000001', + 'apm-7.10.2-profile-000001', + 'apm-7.10.2-profile-000002', + 'apm-7.10.2-profile-000003', + 'apm-7.10.2-profile-000004', + '.ds-cluster_telemetry_elastic-2024.02.06-000050', + 'apm-7.10.2-profile-000005', + 'apm-7.10.2-profile-000006', + 'apm-7.10.2-profile-000008', + 'apm-7.10.2-profile-000009', + '.ds-cluster_telemetry_elastic-2024.01.07-000048', + 'apm-7.10.2-profile-000007', + 'alerts-detections', + 'apm-7.10.2-transaction-000002', + 'apm-7.10.2-transaction-000004', + 'apm-7.11.0-profile-000020', + 'apm-7.11.0-profile-000021', + 'apm-7.11.0-profile-000022', + 'apm-7.10.2-transaction-000003', + 'apm-7.10.2-transaction-000009', + 'apm-7.10.2-transaction-000006', + 'apm-7.10.2-transaction-000005', + 'apm-7.10.2-transaction-000008', + 'apm-7.10.2-transaction-000007', + '.ds-logs-elastic_agent.metricbeat-default-2023.06.22-000001', + '.ds-logs-ingest-vt-reports-test-default-2023.07.23-000002', + 'apm-7.10.2-transaction-000010', + 'apm-7.10.2-profile-000010', + 'apm-7.10.2-profile-000011', + 'apm-7.10.2-profile-000012', + 'apm-7.10.2-profile-000013', + 'apm-7.10.2-profile-000014', + 'apm-7.10.2-profile-000015', + 'apm-7.10.2-profile-000016', + 'apm-7.10.2-profile-000017', + 'apm-7.10.2-profile-000018', + 'apm-7.10.2-profile-000019', + '.kibana_task_manager_7.16.2_001', + 'apm-7.10.2-transaction-000013', + 'apm-7.10.2-transaction-000012', + 'apm-7.10.2-transaction-000015', + 'apm-7.10.2-transaction-000014', + 'apm-7.10.2-transaction-000011', + 'apm-7.10.2-transaction-000017', + 'apm-7.10.2-transaction-000016', + 'apm-7.10.2-transaction-000019', + 'apm-7.10.2-transaction-000018', + '.fleet-policies-7', + 'apm-7.12.0-transaction-000009', + 'apm-7.12.0-transaction-000005', + 'apm-7.12.0-transaction-000006', + 'apm-7.12.0-transaction-000007', + 'apm-7.12.0-transaction-000008', + 'apm-7.12.0-transaction-000001', + '.internal.alerts-observability.metrics.alerts-default-000001', + 'apm-7.15.1-error-000002', + 'apm-7.15.1-error-000001', + '.ds-telemetry_usage-2024.04.02-000042', + 'apm-7.10.2-profile-000020', + 'apm-7.10.2-profile-000021', + 'apm-7.10.2-profile-000022', + 'apm-7.10.2-profile-000023', + '.ds-yaas_alert_telemetry-2022.08.10-000011', + 'apm-7.10.2-transaction-000023', + 'apm-7.10.2-transaction-000020', + 'apm-7.10.2-transaction-000022', + 'apm-7.10.2-transaction-000021', + 'apm-7.15.1-error-000004', + 'apm-7.15.1-error-000003', + 'apm-7.15.1-error-000006', + 'apm-7.15.1-error-000005', + 'apm-7.15.1-error-000008', + 'apm-7.15.1-error-000007', + 'apm-7.15.1-error-000009', + 'apm-7.12.0-transaction-000002', + 'apm-7.12.0-transaction-000003', + 'apm-7.12.0-transaction-000004', + 'apm-7.12.0-transaction-000016', + 'apm-7.12.0-transaction-000017', + 'apm-7.12.0-transaction-000018', + 'apm-7.12.0-transaction-000019', + 'apm-7.12.0-transaction-000012', + 'apm-7.12.0-transaction-000013', + '.ds-telemetry_usage-2024.03.03-000040', + 'apm-7.15.1-error-000010', + 'onweek_rules_browser_index', + 'apm-7.15.1-error-000011', + '.ds-yaas_alert_telemetry-2022.07.11-000010', + 'apm-7.15.1-error-000013', + 'apm-7.15.1-error-000012', + '.ds-alert_timelines_elastic-2024.06.27-000057', + 'apm-7.12.0-transaction-000014', + 'apm-7.12.0-transaction-000015', + 'apm-7.12.0-transaction-000010', + 'apm-7.12.0-transaction-000011', + 'apm-7.16.1-onboarding-2021.12.16', + 'apm-7.16.1-onboarding-2021.12.15', + 'apm-7.12.0-transaction-000020', + '.ds-alert_timelines_elastic-2024.05.28-000053', + 'apm-7.12.0-error-000001', + 'apm-7.12.0-error-000002', + 'apm-7.12.0-error-000003', + 'apm-7.12.0-error-000004', + 'apm-7.12.0-error-000005', + 'apm-7.12.0-error-000006', + 'apm-7.12.0-error-000007', + 'apm-7.12.0-error-000008', + 'apm-7.12.0-error-000009', + 'failure-alert_telemetry_elastic', + '.ds-logs-elastic_agent-default-2023.06.22-000001', + '.ds-metrics-elastic_agent.filebeat-default-2023.11.27-000004', + 'apm-7.10.0-profile-000001', + 'apm-7.10.0-profile-000002', + 'apm-7.10.0-profile-000003', + 'apm-7.10.0-profile-000004', + 'apm-7.10.0-profile-000005', + 'apm-7.10.0-profile-000006', + 'apm-7.10.0-profile-000007', + 'apm-7.10.0-profile-000008', + 'apm-7.10.0-profile-000009', + 'apm-7.12.0-onboarding-2021.03.23', + 'apm-7.12.0-error-000010', + 'apm-7.12.0-error-000011', + 'apm-7.12.0-error-000012', + 'apm-7.12.0-error-000013', + 'apm-7.12.0-error-000014', + 'apm-7.12.0-error-000015', + 'apm-7.12.0-error-000016', + 'apm-7.12.0-error-000017', + 'apm-7.12.0-error-000018', + '.ds-alert_timelines_elastic_v2-2022.10.20-000002', + 'apm-7.12.0-error-000019', + '.ds-alert_telemetry_elastic-2024.09.25-000087', + '.ds-alert_telemetry_elastic-2024.08.26-000078', + 'dead-letter', + '.ds-detections_alert_telemetry_elastic-2024.06.01-000064', + 'apm-7.10.0-profile-000010', + 'apm-7.10.0-profile-000011', + 'apm-7.10.0-profile-000012', + 'apm-7.10.0-profile-000013', + 'apm-7.10.0-profile-000014', + 'apm-7.10.0-profile-000015', + 'apm-7.10.0-profile-000016', + 'apm-7.10.0-profile-000017', + 'apm-7.10.0-profile-000018', + 'apm-7.10.0-profile-000019', + 'data-dictionaries', + 'apm-7.12.0-error-000020', + '.ds-alert_telemetry_elastic-2024.07.27-000075', + '.ds-detections_alert_telemetry_elastic-2024.05.02-000061', + 'apm-7.10.0-profile-000020', + 'apm-7.10.0-profile-000021', + 'apm-7.10.0-profile-000022', + 'apm-7.10.0-profile-000023', + 'apm-7.10.0-profile-000024', + 'apm-7.17.0-error-000001', + 'shrink-e4jt-.ds-alert_timelines_elastic-2023.11.21-000034', + 'apm-7.17.0-error-000003', + 'apm-7.17.0-error-000002', + 'apm-7.17.0-error-000005', + 'apm-7.17.0-error-000006', + 'apm-7.17.0-error-000007', + 'apm-7.17.0-error-000008', + 'apm-7.17.0-error-000009', + 'apm-7.17.0-error-000004', + 'apm-7.16.1-span-000001', + 'apm-7.16.1-span-000002', + 'apm-7.16.1-span-000003', + 'apm-7.16.1-span-000004', + 'apm-7.16.1-span-000005', + 'apm-7.16.1-span-000006', + 'apm-7.16.1-span-000007', + 'apm-7.16.1-span-000008', + 'apm-7.16.1-span-000009', + '.ds-insights_telemetry_elastic-2024.02.09-000040', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.11.27-000004', + '.ds-logs-elastic_agent.filebeat-default-2023.06.22-000001', + 'ia-signatures', + 'apm-7.16.2-onboarding-2021.12.20', + 'apm-7.14.2-transaction-000001', + 'apm-7.14.2-transaction-000002', + 'apm-7.14.2-transaction-000003', + 'apm-7.14.2-transaction-000005', + 'apm-7.14.2-transaction-000006', + 'apm-7.14.2-transaction-000007', + 'apm-7.14.2-transaction-000008', + 'apm-7.14.2-transaction-000009', + '.ds-security_lists_telemetry_elastic-2023.10.18-000040', + 'apm-7.14.2-transaction-000004', + 'apm-7.16.1-span-000011', + 'apm-7.10.0-error-000001', + 'apm-7.10.0-error-000002', + 'apm-7.10.0-error-000003', + 'apm-7.10.0-error-000004', + 'apm-7.10.0-error-000005', + 'apm-7.10.0-error-000006', + 'apm-7.10.0-error-000007', + 'apm-7.10.0-error-000008', + 'apm-7.10.0-error-000009', + '.ds-insights_telemetry_elastic-2023.09.12-000030', + '.ds-detonate_results_telemetry-2022.04.20-000003', + 'apm-7.12.0-span-000001', + 'apm-7.12.0-span-000005', + 'apm-7.12.0-span-000006', + 'apm-7.12.0-span-000007', + 'apm-7.12.0-span-000008', + 'apm-7.12.0-span-000002', + 'apm-7.12.0-span-000003', + 'apm-7.12.0-span-000004', + 'apm-7.12.0-span-000009', + 'apm-7.14.2-transaction-000012', + 'apm-7.14.2-transaction-000013', + 'apm-7.14.2-transaction-000010', + 'apm-7.14.2-transaction-000011', + 'apm-7.16.1-span-000010', + 'apm-7.10.0-span-000001', + 'apm-7.10.0-span-000002', + '.ds-sample-tags-tracking-2022.09.06-000009', + 'apm-7.10.0-error-000010', + 'apm-7.10.0-error-000011', + 'apm-7.10.0-error-000012', + 'apm-7.10.0-error-000014', + 'apm-7.10.0-error-000015', + 'apm-7.10.0-error-000013', + 'apm-7.10.0-error-000016', + 'apm-7.10.0-error-000018', + 'apm-7.10.0-error-000019', + 'apm-7.10.0-error-000017', + 'apm-7.10.0-span-000009', + 'apm-7.10.0-span-000007', + 'apm-7.10.0-span-000008', + 'apm-7.10.0-span-000005', + 'apm-7.10.0-span-000006', + 'apm-7.10.0-span-000003', + 'apm-7.10.0-span-000004', + 'apm-7.12.0-span-000016', + 'apm-7.12.0-span-000017', + 'apm-7.12.0-span-000018', + 'apm-7.12.0-span-000019', + 'apm-7.12.0-span-000012', + 'apm-7.12.0-span-000013', + 'apm-7.12.0-span-000014', + 'apm-7.12.0-span-000015', + 'apm-7.12.0-span-000010', + '.ds-sample-tags-tracking-2022.08.07-000008', + 'apm-7.10.0-span-000010', + 'apm-7.10.0-span-000011', + 'apm-7.10.0-span-000012', + 'apm-7.10.0-error-000020', + 'apm-7.10.0-error-000021', + 'apm-7.10.0-error-000022', + 'apm-7.10.0-error-000023', + 'apm-7.10.0-error-000024', + 'apm-7.10.0-span-000013', + 'apm-7.12.0-span-000011', + 'apm-7.10.0-span-000018', + 'apm-7.12.0-span-000020', + '.ds-detonate_results_telemetry-2022.06.19-000007', + 'apm-7.10.0-span-000019', + 'apm-7.10.0-span-000016', + 'apm-7.10.0-span-000017', + 'apm-7.10.0-span-000014', + 'apm-7.10.0-span-000015', + '.ds-inline_lookup_decision_tracking-2022.08.04-000008', + '.transform-internal-007', + '.transform-internal-006', + '.transform-internal-005', + '.ds-inline_lookup_decision_tracking-2022.09.03-000009', + '.ds-kibana-snapshot-2024.06.23-000052', + '.ds-inline_lookup_decision_tracking-2022.07.05-000007', + '.ds-metrics-elastic_agent.filebeat-default-2023.06.22-000001', + 'event-type-index', + '.ds-sample-tags-tracking-2022.07.08-000007', + '.ds-telemetry_usage-2024.02.02-000038', + 'apm-7.10.0-span-000020', + 'apm-7.10.0-span-000021', + 'apm-7.10.0-span-000022', + 'apm-7.10.0-span-000023', + 'apm-7.10.0-span-000024', + '.ds-cluster_telemetry_elastic-2023.12.08-000046', + '.ds-yaas_alert_telemetry-2022.04.12-000007', + '.kibana_task_manager_7.14.2_001', + '.ds-metrics-elastic_agent.elastic_agent-default-2023.06.22-000001', + '.ds-logs-vt_reports_gold_table_export-default-2023.07.22-000002', + 'apm-7.16.2-metric-000001', + 'apm-7.16.2-metric-000002', + 'apm-7.16.2-metric-000003', + 'apm-7.16.2-metric-000004', + 'apm-7.16.2-metric-000005', + 'apm-7.16.2-metric-000006', + 'apm-7.16.2-metric-000007', + 'apm-7.16.2-metric-000008', + 'apm-7.16.2-metric-000009', + '.internal.alerts-observability.threshold.alerts-default-000001', + 'apm-7.17.0-onboarding-2022.02.24', + '.ds-endpoint_metadata_telemetry_elastic-2023.12.20-000051', + '.internal.alerts-observability.logs.alerts-default-000001', + '.ds-machine_learning_cluster_metrics-2022.11.07-000010', + 'apm-7.16.2-metric-000010', + 'apm-7.16.2-metric-000011', + '.ds-osquery_packs_telemetry_elastic-2022.03.24-000001', + 'shrink-yvq6-.ds-alert_timelines_elastic-2023.10.22-000031', + 'apm-7.14.2-onboarding-2021.10.13', + 'ia-cs_extract', + 'idx_processed_security-stats', + '.ds-insights_telemetry_elastic-2023.11.11-000034', + 'machine_learning_cluster_metric', + '.ds-insights_telemetry_elastic-2024.06.08-000048', + '.kibana-observability-ai-assistant-kb-000001', + '.ds-insights_telemetry_elastic-2023.10.12-000032', + 'detection_rule_cluster_metric', + '.ds-insights_telemetry_elastic-2024.05.09-000046', + '.ds-inline_lookup_decision_tracking-2022.11.02-000011', + '.internal.alerts-security.alerts-default-000001', + '.ds-sample-tags-tracking-2022.11.05-000011', + 'apm-7.14.2-metric-000001', + 'apm-7.14.2-metric-000002', + 'apm-7.14.2-metric-000003', + 'apm-7.14.2-metric-000004', + 'data-dict-index', + 'apm-7.14.2-metric-000005', + 'apm-7.14.2-metric-000006', + 'apm-7.17.0-transaction-000001', + 'apm-7.17.0-transaction-000002', + 'apm-7.17.0-transaction-000003', + 'apm-7.17.0-transaction-000004', + 'apm-7.17.0-transaction-000005', + 'apm-7.17.0-transaction-000006', + 'apm-7.17.0-transaction-000007', + 'apm-7.17.0-transaction-000008', + 'apm-7.17.0-transaction-000009', + 'apm-7.14.2-metric-000009', + 'apm-7.14.2-metric-000007', + 'apm-7.14.2-metric-000008', + 'idx_processed_security-telemetry-usage', + '.ds-inline_lookup_decision_tracking-2022.10.03-000010', + 'shrink-imua-.ds-alert_timelines_elastic-2023.09.22-000028', + 'apm-7.15.1-span-000003', + 'apm-7.15.1-span-000004', + 'apm-7.15.1-span-000005', + 'apm-7.15.1-span-000006', + 'apm-7.15.1-span-000007', + 'apm-7.15.1-span-000008', + 'apm-7.15.1-span-000009', + 'apm-7.15.1-span-000001', + 'apm-7.15.1-span-000002', + 'apm-7.14.2-metric-000010', + 'apm-7.14.2-metric-000012', + 'apm-7.14.2-metric-000013', + 'apm-7.14.2-metric-000011', + '.ds-sample-tags-tracking-2022.10.06-000010', + '.ds-metrics-fleet_server.agent_versions-default-2024.07.20-000002', + 'apm-7.11.0-onboarding-2021.02.10', + 'apm-7.15.1-span-000010', + 'apm-7.15.1-span-000011', + 'apm-7.15.1-span-000012', + 'apm-7.15.1-span-000013', + 'logs-index_pattern_placeholder', + 'awesome-test', + '.ds-yaas_alert_telemetry-2022.10.09-000013', + 'test-vt-reports-export', + '.ds-task_metrics_elastic-2023.09.18-000005', + '.fleet-agents-7', + '.ds-security_lists_telemetry_elastic-2023.09.18-000038', + '.ds-inline_lookup_decision_tracking-2022.04.06-000004', + 'shrink-abnc-.ds-alert_telemetry_elastic-2023.09.27-000037', + 'shrink-5phn-.ds-detections_alert_telemetry_elastic-2023.11.04-000039', + '.ds-yaas_alert_telemetry-2021.11.13-000002', + '.ds-telemetry_usage-2024.05.02-000044', + 'apm-7.11.0-metric-000001', + 'apm-7.11.0-metric-000002', + 'apm-7.11.0-metric-000003', + 'apm-7.11.0-metric-000004', + 'apm-7.11.0-metric-000005', + 'apm-7.11.0-metric-000006', + 'apm-7.11.0-metric-000007', + 'apm-7.11.0-metric-000008', + 'apm-7.11.0-metric-000009', + '.ds-yaas_alert_telemetry-2021.10.14-000001', + '.ds-detections_alert_telemetry_elastic-2024.08.30-000073', + 'apm-7.15.1-onboarding-2021.10.19', + 'apm-7.11.0-metric-000010', + 'apm-7.11.0-metric-000011', + 'apm-7.11.0-metric-000012', + 'apm-7.11.0-metric-000013', + 'apm-7.11.0-metric-000014', + 'apm-7.11.0-metric-000015', + 'apm-7.11.0-metric-000016', + 'apm-7.11.0-metric-000017', + 'apm-7.11.0-metric-000018', + 'apm-7.11.0-metric-000019', + '.ds-detections_alert_telemetry_elastic-2024.07.31-000070', + '.ds-telemetry_usage-2023.12.04-000034', + '.kibana_7.17.0_001', + 'apm-7.11.0-metric-000021', + 'apm-7.11.0-metric-000022', + 'apm-7.11.0-metric-000020', + 'detection-rules-test', + '.ds-v2_detections_alert_telemetry_elastic-2022.09.11-000002', + 'apm-7.16.1-metric-000001', + 'apm-7.16.1-metric-000002', + 'apm-7.16.1-metric-000003', + 'apm-7.16.1-metric-000004', + 'apm-7.16.1-metric-000005', + 'apm-7.16.1-metric-000006', + 'apm-7.16.1-metric-000007', + 'apm-7.16.1-metric-000008', + 'apm-7.16.1-metric-000009', + '.ds-endpoint_metadata_telemetry_elastic-2024.04.18-000062', + 'machine_learning_cluster_detail', + '.items-default-000001', + 'apm-7.10.0-onboarding-2020.12.03', + 'apm-7.16.1-metric-000010', + 'apm-7.16.1-metric-000011', + '.ds-endpoint_metadata_telemetry_elastic-2024.03.19-000060', + 'apm-7.10.0-onboarding-2020.12.04', + 'apm-7.14.2-profile-000001', + 'apm-7.14.2-profile-000002', + 'apm-7.14.2-profile-000003', + 'apm-7.14.2-profile-000004', + 'apm-7.14.2-profile-000005', + 'apm-7.14.2-profile-000006', + 'apm-7.14.2-profile-000007', + 'apm-7.14.2-profile-000008', + 'apm-7.14.2-profile-000009', + '.ds-logs-elastic_agent.filebeat-default-2023.08.21-000003', + 'detection_rule_cluster_detail', + '.ds-alert_telemetry_unified-2022.11.10-000048', + '.kibana_7.15.1_001', + 'apm-7.16.2-error-000001', + 'apm-7.16.2-error-000002', + 'apm-7.16.2-error-000003', + 'apm-7.16.2-error-000004', + 'apm-7.16.2-error-000005', + 'apm-7.16.2-error-000006', + 'apm-7.16.2-error-000007', + 'apm-7.16.2-error-000008', + 'apm-7.16.2-error-000009', + 'apm-7.14.2-profile-000010', +]; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts index edbd5d21e3aea..107ca457b2cfb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts @@ -8,7 +8,7 @@ import axios from 'axios'; import * as rx from 'rxjs'; import _, { cloneDeep } from 'lodash'; -import type { Logger, LogMeta } from '@kbn/core/server'; +import type { AnalyticsServiceSetup, EventTypeOpts, Logger, LogMeta } from '@kbn/core/server'; import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-plugin/server'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; import type { ITelemetryReceiver } from './receiver'; @@ -17,7 +17,7 @@ import { type QueueConfig, type RetryConfig, } from './async_sender.types'; -import { TelemetryChannel, TelemetryCounter } from './types'; +import { type Nullable, TelemetryChannel, TelemetryCounter } from './types'; import * as collections from './collections_helpers'; import { CachedSubject, retryOnError$ } from './rxjs_helpers'; import { SenderUtils } from './sender_helpers'; @@ -55,6 +55,8 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { private telemetryUsageCounter?: IUsageCounter; private senderUtils: SenderUtils | undefined; + private analytics: Nullable; + constructor(logger: Logger) { this.logger = newTelemetryLogger(logger.get('telemetry_events.async_sender')); } @@ -64,7 +66,8 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { fallbackQueueConfig: QueueConfig, telemetryReceiver: ITelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, - telemetryUsageCounter?: IUsageCounter + telemetryUsageCounter?: IUsageCounter, + analytics?: AnalyticsServiceSetup ): void { this.logger.l(`Setting up ${AsyncTelemetryEventsSender.name}`); @@ -77,6 +80,7 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { this.telemetryReceiver = telemetryReceiver; this.telemetrySetup = telemetrySetup; this.telemetryUsageCounter = telemetryUsageCounter; + this.analytics = analytics; this.updateStatus(ServiceStatus.CONFIGURED); } @@ -201,6 +205,13 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { } } + public reportEBT(eventTypeOpts: EventTypeOpts, eventData: T): void { + if (!this.analytics) { + throw Error('analytics is unavailable'); + } + this.analytics.reportEvent(eventTypeOpts.eventType, eventData as object); + } + // internal methods private queue$( upstream$: rx.Observable, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts index 249493cfdbfc8..89ce55d102859 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.types.ts @@ -6,6 +6,7 @@ */ import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-plugin/server'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; +import type { AnalyticsServiceSetup, EventTypeOpts } from '@kbn/core-analytics-server'; import { type TelemetryChannel } from './types'; import type { ITelemetryReceiver } from './receiver'; @@ -20,7 +21,8 @@ export interface IAsyncTelemetryEventsSender { fallbackQueueConfig: QueueConfig, telemetryReceiver: ITelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, - telemetryUsageCounter?: IUsageCounter + telemetryUsageCounter?: IUsageCounter, + analytics?: AnalyticsServiceSetup ) => void; start: (telemetryStart?: TelemetryPluginStart) => void; stop: () => Promise; @@ -28,6 +30,7 @@ export interface IAsyncTelemetryEventsSender { simulateSend: (channel: TelemetryChannel, events: unknown[]) => string[]; updateQueueConfig: (channel: TelemetryChannel, config: QueueConfig) => void; updateDefaultQueueConfig: (config: QueueConfig) => void; + reportEBT: (eventTypeOpts: EventTypeOpts, eventData: T) => void; } /** diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts index 3d67d6cd22b17..987c088d289fe 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.test.ts @@ -4,7 +4,14 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { chunked, chunkedBy } from './collections_helpers'; +import { stagingIndices } from './__mocks__/staging_indices'; +import { + type CommonPrefixesConfig, + chunked, + chunkedBy, + findCommonPrefixes, + chunkStringsByMaxLength, +} from './collections_helpers'; describe('telemetry.utils.chunked', () => { it('should chunk simple case', async () => { @@ -79,3 +86,105 @@ describe('telemetry.utils.chunkedBy', () => { expect(output).toEqual([['aaaa']]); }); }); + +describe('telemetry.utils.findCommonPrefixes', () => { + it('should find common prefixes in simple case', async () => { + const indices = ['aaa', 'b', 'aa']; + const config: CommonPrefixesConfig = { + maxPrefixes: 10, + maxGroupSize: 10, + minPrefixSize: 1, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(2); + expect(output.find((v, _) => v.parts.length === 1 && v.parts[0] === 'a')?.indexCount).toEqual( + 2 + ); + expect(output.find((v, _) => v.parts.length === 1 && v.parts[0] === 'b')?.indexCount).toEqual( + 1 + ); + }); + + it('should find common prefixes with different minPrefixSize', async () => { + const indices = ['.ds-AA-0001', '.ds-AA-0002', '.ds-BB-0003']; + const config: CommonPrefixesConfig = { + maxPrefixes: 10, + maxGroupSize: 3, + minPrefixSize: 5, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(2); + expect( + output.find((v, _) => v.parts.length === 1 && v.parts[0] === '.ds-A')?.indexCount + ).toEqual(2); + expect( + output.find((v, _) => v.parts.length === 1 && v.parts[0] === '.ds-B')?.indexCount + ).toEqual(1); + }); + + it('should discard extra indices', async () => { + const indices = ['aaa', 'aaaaaa', 'aa']; + const config: CommonPrefixesConfig = { + maxPrefixes: 1, + maxGroupSize: 2, + minPrefixSize: 3, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(1); + expect(output.find((v, _) => v.parts.length === 1 && v.parts[0] === 'aaa')?.indexCount).toEqual( + 2 + ); + }); + + it('should group many indices', async () => { + const indices = stagingIndices; + const config: CommonPrefixesConfig = { + maxPrefixes: 8, + maxGroupSize: 100, + minPrefixSize: 3, + }; + + const output = findCommonPrefixes(indices, config); + + expect(output).toHaveLength(config.maxPrefixes); + expect(output.map((v, _) => v.indexCount).reduce((acc, i) => acc + i, 0)).toBe(indices.length); + }); +}); + +describe('telemetry.utils.splitIndicesByNameLength', () => { + it('should chunk simple case', async () => { + const input = ['aa', 'b', 'ccc', 'ddd']; + const output = chunkStringsByMaxLength(input, 5); + expect(output).toEqual([['aa', 'b'], ['ccc'], ['ddd']]); + }); + + it('should chunk with remainder', async () => { + const input = ['aaa', 'b']; + const output = chunkStringsByMaxLength(input, 10); + expect(output).toEqual([['aaa', 'b']]); + }); + + it('should chunk with empty list', async () => { + const input: string[] = []; + const output = chunkStringsByMaxLength(input, 3); + expect(output).toEqual([]); + }); + + it('should chunk with single element smaller than max weight', async () => { + const input = ['aa']; + const output = chunkStringsByMaxLength(input, 3); + expect(output).toEqual([['aa']]); + }); + + it('should chunk with single element bigger than max weight', async () => { + const input = ['aaaa', 'bb']; + const output = chunkStringsByMaxLength(input, 4); + expect(output).toEqual([['aaaa'], ['bb']]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts index a104ea1d55bf8..3d945667fd3cb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/collections_helpers.ts @@ -63,3 +63,153 @@ class Chunked { return this.chunks.filter((chunk) => chunk.length > 0); } } + +export interface CommonPrefixesConfig { + maxPrefixes: number; + maxGroupSize: number; + minPrefixSize: number; +} + +interface TrieNode { + char: string; + prefix: string; + children: { [key: string]: TrieNode }; + count: number; + isEnd: boolean; + id: number; +} + +interface Group { + parts: string[]; + indexCount: number; +} + +function newTrieNode(char: string = '', prefix: string = '', id: number = 0): TrieNode { + return { + char, + children: {}, + count: 0, + id, + isEnd: false, + prefix, + }; +} + +/** + * Finds and groups common prefixes from a list of strings. + * + * @param {string[]} indices - An array of strings from which common prefixes will be extracted. + * @param {CommonPrefixesConfig} config - A configuration object that defines the rules for grouping. + * + * The `config` object contains the following properties: + * - maxGroupSize {number}: The maximum number of indices allowed in a group. + * - maxPrefixes {number}: The maximum number of prefix groups to return. + * - minPrefixSize {number}: The minimum length of a prefix required to form a group. It avoid cases like returning + * a single character prefix, e.g., ['.ds-...1', '.ds-....2', ....] -> returns a single group '.' + * + * @returns {Group[]} - An array of groups where each group contains a list of prefix parts and the count of indices that share that prefix. + * + * Example usage: + * + * ```typescript + * const indices = ['apple', 'appetizer', 'application', 'banana', 'band', 'bandage']; + * const config = { + * maxGroupSize: 5, + * maxPrefixes: 3, + * minPrefixSize: 3 + * }; + * + * const result = findCommonPrefixes(indices, config); + * //result = [ + * // { parts: [ 'ban' ], indexCount: 3 }, + * // { parts: [ 'app' ], indexCount: 3 } + * //] + * ``` + */ + +export function findCommonPrefixes(indices: string[], config: CommonPrefixesConfig): Group[] { + const idCounter = function* (): Generator { + let id = 0; + while (true) { + yield id++; + } + }; + + const idGen = idCounter(); + + const root = newTrieNode('', '', idGen.next().value); + for (const index of indices) { + let node = root; + node.count++; + for (const char of index) { + if (!node.children[char]) { + node.children[char] = newTrieNode(char, node.prefix + char, idGen.next().value); + } + node = node.children[char]; + node.count++; + } + node.isEnd = true; + } + + const nodes = [root]; + const prefixes: Group[] = []; + + while (nodes.length > 0) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const node = nodes.pop()!; + if ( + (node.count <= config.maxGroupSize && node.prefix.length >= config.minPrefixSize) || + (Object.keys(node.children).length === 0 && node.prefix.length >= config.minPrefixSize) + ) { + const group: Group = { + parts: [node.prefix], + indexCount: node.count, + }; + prefixes.push(group); + } else { + for (const child of Object.values(node.children)) { + nodes.push(child); + } + } + } + + if (prefixes.length > config.maxPrefixes) { + prefixes.sort((a, b) => a.indexCount - b.indexCount); + + while (prefixes.length > config.maxPrefixes) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const g1 = prefixes.shift()!; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const g2 = prefixes.shift()!; + const mergedGroup: Group = { + parts: g1.parts.concat(g2.parts), + indexCount: g1.indexCount + g2.indexCount, + }; + prefixes.push(mergedGroup); + prefixes.sort((a, b) => a.indexCount - b.indexCount); + } + } + + return prefixes; +} + +/** + * Splits an array of strings into chunks where the total length of strings in each chunk + * does not exceed the specified `maxLength`. + * + * @param strings - An array of strings to be chunked. + * @param maxLength - The maximum total length allowed for strings in each chunk. Defaults to 1024. + * @returns A two-dimensional array where each inner array is a chunk of strings. + * + * @example + * ```typescript + * const strings = ["hello", "world", "this", "is", "a", "test"]; + * const chunks = chunkStringsByMaxLength(strings, 10); + * console.log(chunks); + * // Output: [["hello", "world"], ["this", "is"], ["a", "test"]] + * ``` + */ +export function chunkStringsByMaxLength(strings: string[], maxLength: number = 3072): string[][] { + // plus 1 for the comma separator + return chunkedBy(strings, maxLength, (index) => index.length + 1); +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts index 691b0daae8f2c..963addde2afb4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts @@ -6,7 +6,11 @@ */ import os from 'os'; -import type { PaginationConfiguration, TelemetrySenderChannelConfiguration } from './types'; +import type { + IndicesMetadataConfiguration, + PaginationConfiguration, + TelemetrySenderChannelConfiguration, +} from './types'; class TelemetryConfigurationDTO { private readonly DEFAULT_TELEMETRY_MAX_BUFFER_SIZE = 100; @@ -21,6 +25,15 @@ class TelemetryConfigurationDTO { max_page_size_bytes: Math.min(os.totalmem() * 0.02, 80 * 1024 * 1024), num_docs_to_sample: 10, }; + private readonly DEFAULT_INDICES_METADATA_CONFIG = { + indices_threshold: 10000, + datastreams_threshold: 1000, + + max_prefixes: 10, // @deprecated + max_group_size: 100, // @deprecated + min_group_size: 5, // @deprecated + }; + private _telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; private _max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH; private _max_endpoint_telemetry_batch = this.DEFAULT_MAX_ENDPOINT_TELEMETRY_BATCH; @@ -31,6 +44,8 @@ class TelemetryConfigurationDTO { [key: string]: TelemetrySenderChannelConfiguration; } = this.DEFAULT_SENDER_CHANNELS; private _pagination_config: PaginationConfiguration = this.DEFAULT_PAGINATION_CONFIG; + private _indices_metadata_config: IndicesMetadataConfiguration = + this.DEFAULT_INDICES_METADATA_CONFIG; public get telemetry_max_buffer_size(): number { return this._telemetry_max_buffer_size; @@ -96,6 +111,14 @@ class TelemetryConfigurationDTO { return this._pagination_config; } + public set indices_metadata_config(paginationConfiguration: IndicesMetadataConfiguration) { + this._indices_metadata_config = paginationConfiguration; + } + + public get indices_metadata_config(): IndicesMetadataConfiguration { + return this._indices_metadata_config; + } + public resetAllToDefault() { this._telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; this._max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH; @@ -104,6 +127,7 @@ class TelemetryConfigurationDTO { this._max_detection_alerts_batch = this.DEFAULT_MAX_DETECTION_ALERTS_BATCH; this._sender_channels = this.DEFAULT_SENDER_CHANNELS; this._pagination_config = this.DEFAULT_PAGINATION_CONFIG; + this._indices_metadata_config = this.DEFAULT_INDICES_METADATA_CONFIG; } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts index 02a39be555110..662704bc59bc9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/event_based/events.ts @@ -11,6 +11,7 @@ import type { ResponseActionsApiCommandNames, } from '../../../../common/endpoint/service/response_actions/constants'; import type { BulkUpsertAssetCriticalityRecordsResponse } from '../../../../common/api/entity_analytics'; +import type { DataStreams, IlmPolicies, IlmsStats, IndicesStats } from '../indices.metadata.types'; export const RISK_SCORE_EXECUTION_SUCCESS_EVENT: EventTypeOpts<{ scoresWritten: number; @@ -271,6 +272,241 @@ export const ALERT_SUPPRESSION_EVENT: EventTypeOpts<{ }, }; +export const TELEMETRY_DATA_STREAM_EVENT: EventTypeOpts = { + eventType: 'telemetry_data_stream_event', + schema: { + items: { + type: 'array', + items: { + properties: { + datastream_name: { + type: 'keyword', + _meta: { description: 'Name of the data stream' }, + }, + indices: { + type: 'array', + items: { + properties: { + index_name: { type: 'date', _meta: { description: 'Index name' } }, + ilm_policy: { type: 'date', _meta: { optional: true, description: 'ILM policy' } }, + }, + }, + _meta: { optional: true, description: 'Indices associated with the data stream' }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + +export const TELEMETRY_INDEX_STATS_EVENT: EventTypeOpts = { + eventType: 'telemetry_index_stats_event', + schema: { + items: { + type: 'array', + items: { + properties: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index being monitored.' }, + }, + query_total: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of search queries executed on the index.', + }, + }, + query_time_in_millis: { + type: 'long', + _meta: { + optional: true, + description: + 'The total time spent on query execution across all search requests, measured in milliseconds.', + }, + }, + docs_count: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of documents currently stored in the index.', + }, + }, + docs_deleted: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents that have been marked as deleted in the index.', + }, + }, + docs_total_size_in_bytes: { + type: 'long', + _meta: { + optional: true, + description: + 'The total size, in bytes, of all documents stored in the index, including storage overhead.', + }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + +export const TELEMETRY_ILM_POLICY_EVENT: EventTypeOpts = { + eventType: 'telemetry_ilm_policy_event', + schema: { + items: { + type: 'array', + items: { + properties: { + policy_name: { + type: 'keyword', + _meta: { description: 'The name of the ILM policy.' }, + }, + modified_date: { + type: 'date', + _meta: { description: 'The date when the ILM policy was last modified.' }, + }, + phases: { + properties: { + cold: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "cold" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "cold" phase of the ILM policy, applied when data is infrequently accessed.', + }, + }, + delete: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "delete" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "delete" phase of the ILM policy, specifying when the index should be removed.', + }, + }, + frozen: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "frozen" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "frozen" phase of the ILM policy, where data is fully searchable but stored with a reduced resource footprint.', + }, + }, + hot: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "hot" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "hot" phase of the ILM policy, applied to actively written and queried data.', + }, + }, + warm: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "warm" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "warm" phase of the ILM policy, used for read-only data that is less frequently accessed.', + }, + }, + }, + _meta: { + description: + 'The different phases of the ILM policy that define how the index is managed over time.', + }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + +export const TELEMETRY_ILM_STATS_EVENT: EventTypeOpts = { + eventType: 'telemetry_ilm_stats_event', + schema: { + items: { + type: 'array', + items: { + properties: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index currently managed by the ILM policy.' }, + }, + phase: { + type: 'keyword', + _meta: { + optional: true, + description: + 'The current phase of the ILM policy that the index is in (e.g., hot, warm, cold, frozen, or delete).', + }, + }, + age: { + type: 'text', + _meta: { + optional: true, + description: + 'The age of the index since its creation, indicating how long it has existed.', + }, + }, + policy_name: { + type: 'keyword', + _meta: { + optional: true, + description: 'The name of the ILM policy applied to this index.', + }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + interface CreateAssetCriticalityProcessedFileEvent { result?: BulkUpsertAssetCriticalityRecordsResponse['stats']; startTime: Date; @@ -457,4 +693,8 @@ export const events = [ ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT, ENTITY_ENGINE_INITIALIZATION_EVENT, ENTITY_STORE_USAGE_EVENT, + TELEMETRY_DATA_STREAM_EVENT, + TELEMETRY_ILM_POLICY_EVENT, + TELEMETRY_ILM_STATS_EVENT, + TELEMETRY_INDEX_STATS_EVENT, ]; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts new file mode 100644 index 0000000000000..fe669c6693d2a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DateTime } from '@elastic/elasticsearch/lib/api/types'; +import type { Nullable } from './types'; + +export interface IlmPolicies { + items: IlmPolicy[]; +} + +export interface IlmPolicy { + policy_name: string; + modified_date: DateTime; + phases: IlmPhases; +} + +export interface IlmPhases { + cold: Nullable; + delete: Nullable; + frozen: Nullable; + hot: Nullable; + warm: Nullable; +} + +export interface IlmPhase { + min_age: string; +} + +export interface IlmsStats { + items: IlmStats[]; +} + +export interface IlmStats { + index_name: string; + phase?: string; + age?: string; + policy_name?: string; +} + +export interface IndicesStats { + items: IndexStats[]; +} + +export interface IndexStats { + index_name: string; + query_total?: number; + query_time_in_millis?: number; + docs_count?: number; + docs_deleted?: number; + docs_total_size_in_bytes?: number; +} + +export interface Index { + index_name: string; + ilm_policy?: string; +} + +export interface DataStreams { + items: DataStream[]; +} +export interface DataStream { + datastream_name: string; + indices?: Index[]; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts index fc195098787dd..66161274fe45c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts @@ -7,7 +7,7 @@ import type { AxiosInstance, AxiosResponse } from 'axios'; import axios, { AxiosHeaders } from 'axios'; -import type { Logger } from '@kbn/core/server'; +import type { EventTypeOpts, Logger } from '@kbn/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; @@ -37,6 +37,9 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { /** Last sent message */ private sentMessages: string[] = []; + /** Last sent EBT events */ + private ebtEventsSent: Array<{ eventType: string; eventData: object }> = []; + /** Logger for this class */ private logger: Logger; @@ -87,6 +90,10 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { return this.sentMessages; } + public getEbtEventsSent(): Array<{ eventType: string; eventData: object }> { + return this.ebtEventsSent; + } + public setup( telemetryReceiver: ITelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, @@ -174,4 +181,12 @@ export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { public updateDefaultQueueConfig(config: QueueConfig): void { this.composite.updateDefaultQueueConfig(config); } + + public reportEBT(eventTypeOpts: EventTypeOpts, eventData: T): void { + this.ebtEventsSent.push({ + eventType: eventTypeOpts.eventType, + eventData: eventData as object, + }); + this.composite.reportEBT(eventTypeOpts, eventData); + } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 4d2ff971eeb62..db023a593f14b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -18,6 +18,7 @@ import type { } from '@kbn/core/server'; import type { AggregationsAggregate, + IlmExplainLifecycleRequest, OpenPointInTimeResponse, SearchRequest, SearchResponse, @@ -38,6 +39,10 @@ import type { SearchHit, SearchRequest as ESSearchRequest, SortResults, + IndicesGetDataStreamRequest, + IndicesStatsRequest, + IlmGetLifecycleRequest, + IndicesGetRequest, } from '@elastic/elasticsearch/lib/api/types'; import type { TransportResult } from '@elastic/elasticsearch'; import type { AgentPolicy, Installation } from '@kbn/fleet-plugin/common'; @@ -87,6 +92,16 @@ import { telemetryConfiguration } from './configuration'; import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants'; import type { TelemetryLogger } from './telemetry_logger'; +import type { + DataStream, + IlmPhase, + IlmPhases, + IlmPolicy, + IlmStats, + Index, + IndexStats, +} from './indices.metadata.types'; +import { chunkStringsByMaxLength } from './collections_helpers'; export interface ITelemetryReceiver { start( @@ -238,6 +253,12 @@ export interface ITelemetryReceiver { setMaxPageSizeBytes(bytes: number): void; setNumDocsToSample(n: number): void; + + getIndices(): Promise; + getDataStreams(): Promise; + getIndicesStats(indices: string[]): AsyncGenerator; + getIlmsStats(indices: string[]): AsyncGenerator; + getIlmsPolicies(ilms: string[]): AsyncGenerator; } export class TelemetryReceiver implements ITelemetryReceiver { @@ -532,7 +553,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { const buckets = endpointMetadataResponse?.aggregations?.endpoint_metadata?.buckets ?? []; return buckets.reduce((cache, endpointAgentId) => { - // const id = endpointAgentId.latest_metadata.hits.hits[0]._id; const doc = endpointAgentId.latest_metadata.hits.hits[0]._source; cache.set(endpointAgentId.key, doc); return cache; @@ -541,7 +561,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public async *fetchDiagnosticAlertsBatch(executeFrom: string, executeTo: string) { - this.logger.debug('Searching diagnostic alerts', { + this.logger.l('Searching diagnostic alerts', { from: executeFrom, to: executeTo, } as LogMeta); @@ -585,10 +605,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { fetchMore = false; } - this.logger.debug('Diagnostic alerts to return', { numOfHits } as LogMeta); + this.logger.l('Diagnostic alerts to return', { numOfHits } as LogMeta); fetchMore = numOfHits > 0 && numOfHits < telemetryConfiguration.telemetry_max_buffer_size; } catch (e) { - this.logger.l('Error fetching alerts', { error: JSON.stringify(e) }); + this.logger.warn('Error fetching alerts', { error_message: e.message } as LogMeta); fetchMore = false; } @@ -761,7 +781,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { executeFrom: string, executeTo: string ) { - this.logger.debug('Searching prebuilt rule alerts from', { + this.logger.l('Searching prebuilt rule alerts from', { executeFrom, executeTo, } as LogMeta); @@ -899,14 +919,14 @@ export class TelemetryReceiver implements ITelemetryReceiver { pitId = response?.pit_id; } - this.logger.debug('Prebuilt rule alerts to return', { alerts: alerts.length } as LogMeta); + this.logger.l('Prebuilt rule alerts to return', { alerts: alerts.length } as LogMeta); yield alerts; } } catch (e) { // to keep backward compatibility with the previous implementation, silent return // once we start using `paginate` this error should be managed downstream - this.logger.l('Error fetching alerts', { error: JSON.stringify(e) }); + this.logger.warn('Error fetching alerts', { error_message: e.message } as LogMeta); return; } finally { await this.closePointInTime(pitId); @@ -930,10 +950,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { try { await this.esClient().closePointInTime({ id: pitId }); } catch (error) { - this.logger.l('Error trying to close point in time', { + this.logger.warn('Error trying to close point in time', { pit: pitId, - error: JSON.stringify(error), - }); + error_message: error.message, + } as LogMeta); } } @@ -1019,7 +1039,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { fetchMore = numOfHits > 0; } catch (e) { - this.logger.l('Error fetching alerts', { error: JSON.stringify(e) }); + this.logger.warn('Error fetching alerts', { error_message: e.message } as LogMeta); fetchMore = false; } @@ -1034,11 +1054,11 @@ export class TelemetryReceiver implements ITelemetryReceiver { try { await this.esClient().closePointInTime({ id: pitId }); } catch (error) { - this.logger.l('Error trying to close point in time', { + this.logger.warn('Error trying to close point in time', { pit: pitId, - error: JSON.stringify(error), + error_message: error.message, keepAlive, - }); + } as LogMeta); } this.logger.l('Timeline alerts to return', { alerts: alertsToReturn.length }); @@ -1232,7 +1252,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return ret.license; } catch (err) { - this.logger.l('failed retrieving license', { error: JSON.stringify(err) }); + this.logger.warn('failed retrieving license', { error_message: err.message } as LogMeta); return undefined; } } @@ -1293,7 +1313,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { yield data; } while (esQuery.search_after !== undefined); } catch (e) { - this.logger.l('Error running paginated query', { error: JSON.stringify(e) }); + this.logger.warn('Error running paginated query', { error_message: e.message } as LogMeta); throw e; } finally { await this.closePointInTime(pit.id); @@ -1320,4 +1340,198 @@ export class TelemetryReceiver implements ITelemetryReceiver { } return this._esClient; } + + public async getIndices(): Promise { + const es = this.esClient(); + + this.logger.l('Fetching indices'); + + const request: IndicesGetRequest = { + index: '*', + expand_wildcards: ['open', 'hidden'], + filter_path: ['*.settings.index.provided_name'], + }; + + return es.indices + .get(request) + .then((indices) => Array.from(Object.keys(indices))) + .catch((error) => { + this.logger.warn('Error fetching indices', { error_message: error } as LogMeta); + throw error; + }); + } + + public async getDataStreams(): Promise { + const es = this.esClient(); + + this.logger.l('Fetching datstreams'); + + const request: IndicesGetDataStreamRequest = { + name: '*', + expand_wildcards: ['open', 'hidden'], + filter_path: ['data_streams.name', 'data_streams.indices'], + }; + + return es.indices + .getDataStream(request) + .then((response) => + response.data_streams.map((ds) => { + return { + datastream_name: ds.name, + indices: + ds.indices?.map((index) => { + return { + index_name: index.index_name, + ilm_policy: index.ilm_policy, + } as Index; + }) ?? [], + } as DataStream; + }) + ) + .catch((error) => { + this.logger.warn('Error fetching datastreams', { error_message: error } as LogMeta); + throw error; + }); + } + + public async *getIndicesStats(indices: string[]) { + const es = this.esClient(); + + this.logger.l('Fetching indices stats'); + + const groupedIndices = chunkStringsByMaxLength(indices); + + this.logger.l('Splitted indices into groups', { + groups: groupedIndices.length, + indices: indices.length, + } as LogMeta); + + for (const group of groupedIndices) { + const request: IndicesStatsRequest = { + index: group, + level: 'indices', + metric: ['docs', 'search', 'store'], + expand_wildcards: ['open', 'hidden'], + filter_path: [ + 'indices.*.total.search.query_total', + 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', + 'indices.*.total.docs.deleted', + 'indices.*.total.store.size_in_bytes', + ], + }; + + try { + const response = await es.indices.stats(request); + for (const [indexName, stats] of Object.entries(response.indices ?? {})) { + yield { + index_name: indexName, + query_total: stats.total?.search?.query_total, + query_time_in_millis: stats.total?.search?.query_time_in_millis, + docs_count: stats.total?.docs?.count, + docs_deleted: stats.total?.docs?.deleted, + docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, + } as IndexStats; + } + } catch (error) { + this.logger.warn('Error fetching indices stats', { error_message: error } as LogMeta); + throw error; + } + } + } + + public async *getIlmsStats(indices: string[]) { + const es = this.esClient(); + + const groupedIndices = chunkStringsByMaxLength(indices); + + this.logger.l('Splitted ilms into groups', { + groups: groupedIndices.length, + indices: indices.length, + } as LogMeta); + + for (const group of groupedIndices) { + const request: IlmExplainLifecycleRequest = { + index: group.join(','), + only_managed: false, + filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], + }; + + const data = await es.ilm.explainLifecycle(request); + + try { + for (const [indexName, stats] of Object.entries(data.indices ?? {})) { + const entry = { + index_name: indexName, + phase: ('phase' in stats && stats.phase) || undefined, + age: ('age' in stats && stats.age) || undefined, + policy_name: ('policy' in stats && stats.policy) || undefined, + } as IlmStats; + + yield entry; + } + } catch (error) { + this.logger.warn('Error fetching ilm stats', { error_message: error } as LogMeta); + throw error; + } + } + } + + public async *getIlmsPolicies(ilms: string[]) { + const es = this.esClient(); + + const phase = (obj: unknown): Nullable => { + let value: Nullable; + if (obj !== null && obj !== undefined && typeof obj === 'object' && 'min_age' in obj) { + value = { + min_age: obj.min_age, + } as IlmPhase; + } + return value; + }; + + const groupedIlms = chunkStringsByMaxLength(ilms); + + this.logger.l('Splitted ilms into groups', { + groups: groupedIlms.length, + ilms: ilms.length, + } as LogMeta); + + for (const group of groupedIlms) { + this.logger.l('Fetching ilm policies'); + const request: IlmGetLifecycleRequest = { + name: group.join(','), + filter_path: [ + '*.policy.phases.cold.min_age', + '*.policy.phases.delete.min_age', + '*.policy.phases.frozen.min_age', + '*.policy.phases.hot.min_age', + '*.policy.phases.warm.min_age', + '*.modified_date', + ], + }; + + const response = await es.ilm.getLifecycle(request); + try { + for (const [policyName, stats] of Object.entries(response ?? {})) { + yield { + policy_name: policyName, + modified_date: stats.modified_date, + phases: { + cold: phase(stats.policy.phases.cold), + delete: phase(stats.policy.phases.delete), + frozen: phase(stats.policy.phases.frozen), + hot: phase(stats.policy.phases.hot), + warm: phase(stats.policy.phases.warm), + } as IlmPhases, + } as IlmPolicy; + } + } catch (error) { + this.logger.warn('Error fetching ilm policies', { + error_message: error.message, + } as LogMeta); + throw error; + } + } + } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 25dd07d4ef986..8e99a9e12981c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -9,7 +9,7 @@ import { cloneDeep } from 'lodash'; import { URL } from 'url'; import { transformDataToNdjson } from '@kbn/securitysolution-utils'; -import type { Logger, LogMeta } from '@kbn/core/server'; +import type { EventTypeOpts, Logger, LogMeta } from '@kbn/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; import type { AxiosInstance } from 'axios'; @@ -88,6 +88,11 @@ export interface ITelemetryEventsSender { * Updates the default queue configuration. */ updateDefaultQueueConfig: (config: QueueConfig) => void; + + /** + * Reports EBT events + */ + reportEBT: (eventTypeOpts: EventTypeOpts, eventData: T) => void; } export class TelemetryEventsSender implements ITelemetryEventsSender { @@ -270,12 +275,16 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { const telemetryUrl = await this.fetchTelemetryPingUrl(); const resp = await axios.get(telemetryUrl, { timeout: 3000 }); if (resp.status === 200) { - this.logger.l('[Security Telemetry] elastic telemetry services are reachable'); + this.logger.debug('Elastic telemetry services are reachable'); return true; } return false; - } catch (_err) { + } catch (e) { + this.logger.warn('Error pinging telemetry services', { + error: e.message, + } as LogMeta); + return false; } } @@ -426,6 +435,10 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { this.getAsyncTelemetrySender().updateDefaultQueueConfig(config); } + public reportEBT(eventTypeOpts: EventTypeOpts, eventData: T): void { + this.getAsyncTelemetrySender().reportEBT(eventTypeOpts, eventData); + } + private getAsyncTelemetrySender(): IAsyncTelemetryEventsSender { if (!this.asyncTelemetrySender) { throw new Error('Telemetry Sender V2 not initialized'); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index 91cb7d881d4c9..7efb2ea1297a6 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -16,6 +16,7 @@ import { createMockTaskMetrics, createMockSecurityTelemetryTask, } from './__mocks__'; +import { newTelemetryLogger } from './helpers'; describe('test security telemetry task', () => { let logger: ReturnType; @@ -66,7 +67,7 @@ describe('test security telemetry task', () => { expect(mockTelemetryTaskConfig.runTask).toHaveBeenCalledWith( telemetryTask.getTaskId(), - logger, + newTelemetryLogger(logger.get('task')), mockTelemetryReceiver, mockTelemetryEventsSender, mockTaskMetrics, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 62d827930e6a9..55c83243adcbc 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -6,7 +6,7 @@ */ import moment from 'moment'; -import type { Logger } from '@kbn/core/server'; +import type { Logger, LogMeta } from '@kbn/core/server'; import type { ConcreteTaskInstance, TaskManagerSetupContract, @@ -15,8 +15,9 @@ import type { import type { ITelemetryReceiver } from './receiver'; import type { ITelemetryEventsSender } from './sender'; import type { ITaskMetricsService } from './task_metrics.types'; -import { tlog } from './helpers'; import { stateSchemaByVersion, emptyState, type LatestTaskStateSchema } from './task_state'; +import { newTelemetryLogger } from './helpers'; +import { type TelemetryLogger } from './telemetry_logger'; export interface SecurityTelemetryTaskConfig { type: string; @@ -49,7 +50,7 @@ export type LastExecutionTimestampCalculator = ( export class SecurityTelemetryTask { private readonly config: SecurityTelemetryTaskConfig; - private readonly logger: Logger; + private readonly logger: TelemetryLogger; private readonly sender: ITelemetryEventsSender; private readonly receiver: ITelemetryReceiver; private readonly taskMetricsService: ITaskMetricsService; @@ -62,7 +63,7 @@ export class SecurityTelemetryTask { taskMetricsService: ITaskMetricsService ) { this.config = config; - this.logger = logger; + this.logger = newTelemetryLogger(logger.get('task')); this.sender = sender; this.receiver = receiver; this.taskMetricsService = taskMetricsService; @@ -122,7 +123,7 @@ export class SecurityTelemetryTask { public start = async (taskManager: TaskManagerStartContract) => { const taskId = this.getTaskId(); - tlog(this.logger, `[task ${taskId}]: attempting to schedule`); + this.logger.debug('Attempting to schedule task', { taskId } as LogMeta); try { await taskManager.ensureScheduled({ id: taskId, @@ -135,30 +136,32 @@ export class SecurityTelemetryTask { params: { version: this.config.version }, }); } catch (e) { - this.logger.error(`[task ${taskId}]: error scheduling task, received ${e.message}`); + this.logger.error('Error scheduling task', { + error: e.message, + } as LogMeta); } }; public runTask = async (taskId: string, executionPeriod: TaskExecutionPeriod) => { - tlog(this.logger, `[task ${taskId}]: attempting to run`); + this.logger.debug('Attempting to run', { taskId } as LogMeta); if (taskId !== this.getTaskId()) { - tlog(this.logger, `[task ${taskId}]: outdated task`); + this.logger.info('outdated task', { taskId } as LogMeta); return 0; } const isOptedIn = await this.sender.isTelemetryOptedIn(); if (!isOptedIn) { - tlog(this.logger, `[task ${taskId}]: telemetry is not opted-in`); + this.logger.info('Telemetry is not opted-in', { taskId } as LogMeta); return 0; } const isTelemetryServicesReachable = await this.sender.isTelemetryServicesReachable(); if (!isTelemetryServicesReachable) { - tlog(this.logger, `[task ${taskId}]: cannot reach telemetry services`); + this.logger.info('Cannot reach telemetry services', { taskId } as LogMeta); return 0; } - tlog(this.logger, `[task ${taskId}]: running task`); + this.logger.debug('Running task', { taskId } as LogMeta); return this.config.runTask( taskId, this.logger, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts index 3306633bcfbb2..966eb0888f45e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task_metrics.ts @@ -29,7 +29,11 @@ export class TaskMetricsService implements ITaskMetricsService { public async end(trace: Trace, error?: Error): Promise { const event = this.createTaskMetric(trace, error); - this.logger.l(`Task ${event.name} complete. Task run took ${event.time_executed_in_ms}ms`); + this.logger.l('Task completed', { + task_name: event.name, + time_executed_in_ms: event.time_executed_in_ms, + error_message: event.error_message, + }); if (telemetryConfiguration.use_async_sender) { this.sender.sendAsync(TelemetryChannel.TASK_METRICS, [event]); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts index 0558971ebf1e3..502c27fd3e2a6 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts @@ -51,7 +51,7 @@ export function createTelemetryConfigurationTaskConfig() { const configArtifact = manifest.data as unknown as TelemetryConfiguration; log.l('Got telemetry configuration artifact', { - artifact: configArtifact, + artifact: configArtifact ?? '', }); telemetryConfiguration.max_detection_alerts_batch = @@ -107,6 +107,11 @@ export function createTelemetryConfigurationTaskConfig() { _receiver.setNumDocsToSample(configArtifact.pagination_config.num_docs_to_sample); } + if (configArtifact.indices_metadata_config) { + log.l('Updating indices metadata configuration'); + telemetryConfiguration.indices_metadata_config = configArtifact.indices_metadata_config; + } + await taskMetricsService.end(trace); log.l('Updated TelemetryConfiguration', { configuration: telemetryConfiguration }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts index d237757616f3e..0cf3c610dbc8b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -16,6 +16,7 @@ import { createTelemetryDiagnosticTimelineTaskConfig } from './timelines_diagnos import { createTelemetryConfigurationTaskConfig } from './configuration'; import { telemetryConfiguration } from '../configuration'; import { createTelemetryFilterListArtifactTaskConfig } from './filterlists'; +import { createTelemetryIndicesMetadataTaskConfig } from './indices.metadata'; export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { return [ @@ -30,5 +31,6 @@ export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] { createTelemetryDiagnosticTimelineTaskConfig(), createTelemetryConfigurationTaskConfig(), createTelemetryFilterListArtifactTaskConfig(), + createTelemetryIndicesMetadataTaskConfig(), ]; } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts new file mode 100644 index 0000000000000..8c90205fa890e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { LogMeta, Logger } from '@kbn/core/server'; +import type { ITelemetryEventsSender } from '../sender'; +import type { ITelemetryReceiver } from '../receiver'; +import type { TaskExecutionPeriod } from '../task'; +import type { ITaskMetricsService } from '../task_metrics.types'; +import { + createUsageCounterLabel, + getPreviousDailyTaskTimestamp, + newTelemetryLogger, +} from '../helpers'; +import { + TELEMETRY_DATA_STREAM_EVENT, + TELEMETRY_ILM_POLICY_EVENT, + TELEMETRY_ILM_STATS_EVENT, + TELEMETRY_INDEX_STATS_EVENT, +} from '../event_based/events'; +import { telemetryConfiguration } from '../configuration'; +import type { + DataStream, + DataStreams, + IlmPolicies, + IlmsStats, + IndicesStats, +} from '../indices.metadata.types'; +import { TelemetryCounter } from '../types'; + +const COUNTER_LABELS = ['security_solution', 'indices-metadata']; + +export function createTelemetryIndicesMetadataTaskConfig() { + const taskType = 'security:indices-metadata-telemetry'; + return { + type: taskType, + title: 'Security Solution Telemetry Indices Metadata task', + interval: '24h', + timeout: '1m', + version: '1.0.0', + getLastExecutionTime: getPreviousDailyTaskTimestamp, + runTask: async ( + taskId: string, + logger: Logger, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, + taskMetricsService: ITaskMetricsService, + taskExecutionPeriod: TaskExecutionPeriod + ) => { + const mdc = { task_id: taskId, task_execution_period: taskExecutionPeriod }; + const log = newTelemetryLogger(logger.get('indices-metadata'), mdc); + const trace = taskMetricsService.start(taskType); + + const taskConfig = telemetryConfiguration.indices_metadata_config; + + const publishDatastreamsStats = (stats: DataStream[]): number => { + const events: DataStreams = { + items: stats, + }; + sender.reportEBT(TELEMETRY_DATA_STREAM_EVENT, events); + log.info(`Sent data streams`, { count: events.items.length } as LogMeta); + return events.items.length; + }; + + const publishIndicesStats = async (indices: string[]): Promise => { + const indicesStats: IndicesStats = { + items: [], + }; + + for await (const stat of receiver.getIndicesStats(indices)) { + indicesStats.items.push(stat); + } + sender.reportEBT(TELEMETRY_INDEX_STATS_EVENT, indicesStats); + log.info(`Sent indices stats`, { count: indicesStats.items.length } as LogMeta); + return indicesStats.items.length; + }; + + const publishIlmStats = async (indices: string[]): Promise> => { + const ilmNames = new Set(); + const ilmsStats: IlmsStats = { + items: [], + }; + + for await (const stat of receiver.getIlmsStats(indices)) { + if (stat.policy_name !== undefined) { + ilmNames.add(stat.policy_name); + ilmsStats.items.push(stat); + } + } + + sender.reportEBT(TELEMETRY_ILM_STATS_EVENT, ilmsStats); + log.info(`Sent ILM stats`, { count: ilmNames.size } as LogMeta); + + return ilmNames; + }; + + const publishIlmPolicies = async (ilmNames: Set): Promise => { + const ilmPolicies: IlmPolicies = { + items: [], + }; + + for await (const policy of receiver.getIlmsPolicies(Array.from(ilmNames.values()))) { + ilmPolicies.items.push(policy); + } + sender.reportEBT(TELEMETRY_ILM_POLICY_EVENT, ilmPolicies); + log.info('Sent ILM policies', { count: ilmPolicies.items.length } as LogMeta); + return ilmPolicies.items.length; + }; + + const incrementCounter = (type: TelemetryCounter, name: string, value: number) => { + const telemetryUsageCounter = sender.getTelemetryUsageCluster(); + telemetryUsageCounter?.incrementCounter({ + counterName: createUsageCounterLabel(COUNTER_LABELS.concat(name)), + counterType: type, + incrementBy: value, + }); + }; + + try { + // 1. Get cluster stats and list of indices and datastreams + const [indices, dataStreams] = await Promise.all([ + receiver.getIndices(), + receiver.getDataStreams(), + ]); + + // 2. Publish datastreams stats + const dsCount = publishDatastreamsStats( + dataStreams.slice(0, taskConfig.datastreams_threshold) + ); + incrementCounter(TelemetryCounter.DOCS_SENT, 'datastreams-stats', dsCount); + + // 3. Get and publish indices stats + const indicesCount: number = await publishIndicesStats( + indices.slice(0, taskConfig.indices_threshold) + ) + .then((count) => { + incrementCounter(TelemetryCounter.DOCS_SENT, 'indices-stats', count); + return count; + }) + .catch((err) => { + log.warn(`Error getting indices stats`, { error: err.message } as LogMeta); + incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'indices-stats', 1); + return 0; + }); + + // 4. Get ILM stats and publish them + const ilmNames = await publishIlmStats(indices.slice(0, taskConfig.indices_threshold)) + .then((names) => { + incrementCounter(TelemetryCounter.DOCS_SENT, 'ilm-stats', names.size); + return names; + }) + .catch((err) => { + log.warn(`Error getting ILM stats`, { error: err.message } as LogMeta); + incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'ilm-stats', 1); + return new Set(); + }); + + // 5. Publish ILM policies + const policyCount = await publishIlmPolicies(ilmNames) + .then((count) => { + incrementCounter(TelemetryCounter.DOCS_SENT, 'ilm-policies', count); + return count; + }) + .catch((err) => { + log.warn(`Error getting ILM policies`, { error: err.message } as LogMeta); + incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'ilm-policies', 1); + return 0; + }); + + log.info(`Sent EBT events`, { + datastreams: dsCount, + ilms: ilmNames.size, + indices: indicesCount, + policies: policyCount, + } as LogMeta); + + await taskMetricsService.end(trace); + + return indicesCount; + } catch (err) { + log.warn(`Error running indices metadata task`, { + error: err.message, + } as LogMeta); + await taskMetricsService.end(trace, err); + return 0; + } + }, + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index eafe04c96a5ab..7741db5721c78 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -465,6 +465,15 @@ export interface TelemetryConfiguration { [key: string]: TelemetrySenderChannelConfiguration; }; pagination_config?: PaginationConfiguration; + indices_metadata_config?: IndicesMetadataConfiguration; +} + +export interface IndicesMetadataConfiguration { + indices_threshold: number; + datastreams_threshold: number; + max_prefixes: number; + max_group_size: number; + min_group_size: number; } export interface PaginationConfiguration { diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 054ecaad5fe21..d8fa5c61ee7f3 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -507,7 +507,8 @@ export class Plugin implements ISecuritySolutionPlugin { DEFAULT_QUEUE_CONFIG, this.telemetryReceiver, plugins.telemetry, - this.telemetryUsageCounter + this.telemetryUsageCounter, + core.analytics ); this.telemetryEventsSender.setup( diff --git a/x-pack/test/common/utils/security_solution/detections_response/index.ts b/x-pack/test/common/utils/security_solution/detections_response/index.ts index 43c2a54900c15..45efc2cee3b4c 100644 --- a/x-pack/test/common/utils/security_solution/detections_response/index.ts +++ b/x-pack/test/common/utils/security_solution/detections_response/index.ts @@ -5,9 +5,10 @@ * 2.0. */ -export * from './rules'; export * from './alerts'; -export * from './delete_all_anomalies'; export * from './count_down_test'; +export * from './delete_all_anomalies'; export * from './route_with_namespace'; +export * from './rules'; +export * from './tasks'; export * from './wait_for'; diff --git a/x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts b/x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts new file mode 100644 index 0000000000000..128f0cfe9b93a --- /dev/null +++ b/x-pack/test/common/utils/security_solution/detections_response/tasks/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './indices_metadata'; +export * from './task_manager'; diff --git a/x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts b/x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts new file mode 100644 index 0000000000000..0c4d90749e423 --- /dev/null +++ b/x-pack/test/common/utils/security_solution/detections_response/tasks/indices_metadata.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; + +const DS_PREFIX = 'testing-datastream'; +const ILM_PREFIX = 'testing-ilm'; + +export const randomDatastream = async (es: Client, policyName?: string): Promise => { + const name = `${DS_PREFIX}-${Date.now()}`; + + let settings = {}; + + if (policyName) { + settings = { + ...settings, + 'index.lifecycle.name': policyName, + }; + } + + const indexTemplateBody = { + index_patterns: [`${DS_PREFIX}-*`], + data_stream: {}, + template: { + settings, + }, + }; + + await es.indices.putIndexTemplate({ + name: DS_PREFIX, + body: indexTemplateBody, + }); + + await es.indices.createDataStream({ name }); + + return name; +}; + +export const randomIlmPolicy = async (es: Client): Promise => { + const name = `${ILM_PREFIX}-${Date.now()}`; + + const policy = { + phases: { + hot: { + actions: { + rollover: { + max_size: '50gb', + max_age: '30d', + }, + }, + }, + warm: { + min_age: '30d', + actions: { + forcemerge: { + max_num_segments: 1, + }, + shrink: { + number_of_shards: 1, + }, + allocate: { + number_of_replicas: 1, + }, + }, + }, + delete: { + min_age: '90d', + actions: { + delete: {}, + }, + }, + }, + }; + + await es.ilm.putLifecycle({ name, policy }); + + return name; +}; + +export const ensureBackingIndices = async (dsName: string, count: number, es: Client) => { + const stats = await es.indices.dataStreamsStats({ name: dsName }); + if (stats.data_streams.length !== 1) { + throw new Error('Data stream not found'); + } + const current = stats.data_streams[0].backing_indices; + + if (current < count) { + for (let i = current; i < count; i++) { + await es.indices.rollover({ alias: dsName }); + } + } else if (current > count) { + throw new Error('Cannot reduce the number of backing indices'); + } +}; + +export const cleanupDatastreams = async (es: Client) => { + await es.indices.deleteDataStream({ name: `${DS_PREFIX}*` }); +}; + +export const cleanupPolicies = async (es: Client) => { + const policies = await es.ilm.getLifecycle({ name: `${ILM_PREFIX}*` }); + + await Promise.all(Object.entries(policies).map(([name, _]) => es.ilm.deleteLifecycle({ name }))); +}; diff --git a/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts b/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts new file mode 100644 index 0000000000000..84015fa7b0b62 --- /dev/null +++ b/x-pack/test/common/utils/security_solution/detections_response/tasks/task_manager.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TaskStatus } from '@kbn/task-manager-plugin/server'; +import { KbnClient } from '@kbn/test'; +import { ToolingLog } from '@kbn/tooling-log'; + +export const taskHasRun = async (taskId: string, kbn: KbnClient, after: Date): Promise => { + const task = await kbn.savedObjects.get({ + type: 'task', + id: taskId, + }); + + const runAt = new Date(task.attributes.runAt); + const status = task.attributes.status; + + return runAt > after && status === TaskStatus.Idle; +}; + +export const launchTask = async ( + taskId: string, + kbn: KbnClient, + logger: ToolingLog, + delayMillis: number = 1_000 +): Promise => { + logger.info(`Launching task ${taskId}`); + const task = await kbn.savedObjects.get({ + type: 'task', + id: taskId, + }); + + const runAt = new Date(Date.now() + delayMillis).toISOString(); + + await kbn.savedObjects.update({ + type: 'task', + id: taskId, + attributes: { + ...task.attributes, + runAt, + scheduledAt: runAt, + status: TaskStatus.Idle, + }, + }); + + logger.info(`Task ${taskId} launched`); + + return new Date(runAt); +}; diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index e8e24f53551e8..2c0a5bc79a8dd 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -164,6 +164,7 @@ export default function ({ getService }: FtrProviderContext) { 'security-solution-ea-asset-criticality-ecs-migration', 'security:endpoint-diagnostics', 'security:endpoint-meta-telemetry', + 'security:indices-metadata-telemetry', 'security:telemetry-configuration', 'security:telemetry-detection-rules', 'security:telemetry-diagnostic-timelines', diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index 25b7a40c9701e..8314b841e2533 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -5,6 +5,8 @@ * 2.0. */ +import path from 'path'; + import { CA_CERT_PATH } from '@kbn/dev-utils'; import { FtrConfigProviderContext, kbnTestConfig, kibanaTestUser } from '@kbn/test'; import { services as baseServices } from './services'; @@ -86,6 +88,11 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s 'riskScoringRoutesEnabled', 'alertSuppressionForSequenceEqlRuleEnabled', ])}`, + `--plugin-path=${path.resolve( + __dirname, + '../../../../../test/analytics/plugins/analytics_ftr_helpers' + )}`, + '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify(PRECONFIGURED_ACTION_CONNECTORS)}`, ...(ssl diff --git a/x-pack/test/security_solution_api_integration/config/ess/services.ts b/x-pack/test/security_solution_api_integration/config/ess/services.ts index e5f66d5c1928a..80d88350e6dd2 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/services.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/services.ts @@ -5,12 +5,14 @@ * 2.0. */ +import { KibanaEBTServerProvider } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; +import { SecuritySolutionESSUtils } from '../services/security_solution_ess_utils'; import { SpacesServiceProvider } from '../../../common/services/spaces'; import { services as essServices } from '../../../api_integration/services'; -import { SecuritySolutionESSUtils } from '../services/security_solution_ess_utils'; export const services = { ...essServices, spaces: SpacesServiceProvider, securitySolutionUtils: SecuritySolutionESSUtils, + kibana_ebt_server: KibanaEBTServerProvider, }; diff --git a/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts b/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts index c2371984512e1..8d3bea36a968b 100644 --- a/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts @@ -4,6 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import path from 'path'; + import { FtrConfigProviderContext } from '@kbn/test'; import { services } from './services'; import { PRECONFIGURED_ACTION_CONNECTORS } from '../shared'; @@ -13,6 +15,7 @@ export interface CreateTestConfigOptions { junit: { reportName: string }; kbnTestServerArgs?: string[]; kbnTestServerEnv?: Record; + suiteTags?: { include?: string[]; exclude?: string[] }; } export function createTestConfig(options: CreateTestConfigOptions) { @@ -22,6 +25,7 @@ export function createTestConfig(options: CreateTestConfigOptions) { ); return { ...svlSharedConfig.getAll(), + suiteTags: options.suiteTags, services: { ...services, }, @@ -32,6 +36,10 @@ export function createTestConfig(options: CreateTestConfigOptions) { '--serverless=security', `--xpack.actions.preconfigured=${JSON.stringify(PRECONFIGURED_ACTION_CONNECTORS)}`, ...(options.kbnTestServerArgs || []), + `--plugin-path=${path.resolve( + __dirname, + '../../../../../test/analytics/plugins/analytics_ftr_helpers' + )}`, ], env: { ...svlSharedConfig.get('kbnTestServer.env'), diff --git a/x-pack/test/security_solution_api_integration/config/serverless/services.ts b/x-pack/test/security_solution_api_integration/config/serverless/services.ts index a7e9be3588842..bfef9e1d4424b 100644 --- a/x-pack/test/security_solution_api_integration/config/serverless/services.ts +++ b/x-pack/test/security_solution_api_integration/config/serverless/services.ts @@ -7,6 +7,7 @@ import { SearchSecureService } from '@kbn/test-suites-serverless/shared/services/search_secure'; import { services as serverlessServices } from '@kbn/test-suites-serverless/api_integration/services'; +import { KibanaEBTServerProvider } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; import { SpacesServiceProvider } from '../../../common/services/spaces'; import { SecuritySolutionServerlessUtils } from '../services/security_solution_serverless_utils'; import { SecuritySolutionServerlessSuperTest } from '../services/security_solution_serverless_supertest'; @@ -17,4 +18,5 @@ export const services = { secureSearch: SearchSecureService, securitySolutionUtils: SecuritySolutionServerlessUtils, supertest: SecuritySolutionServerlessSuperTest, + kibana_ebt_server: KibanaEBTServerProvider, }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts index d13e74724c818..4b22a4b590621 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts @@ -47,40 +47,70 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); removeExtraFieldsFromTelemetryStats(stats); - expect(stats).to.eql({ - detection_rules: [ - [ - { - name: 'security:telemetry-detection-rules', - passed: true, - }, - ], + + expect(stats.detection_rules).to.eql([ + [ + { + name: 'security:telemetry-detection-rules', + passed: true, + }, ], - security_lists: [ - [ - { - name: 'security:telemetry-lists', - passed: true, - }, - ], + ]); + + expect(stats.security_lists).to.eql([ + [ + { + name: 'security:telemetry-lists', + passed: true, + }, ], - endpoints: [ - [ - { - name: 'security:endpoint-meta-telemetry', - passed: true, - }, - ], + ]); + + expect(stats.endpoints).to.eql([ + [ + { + name: 'security:endpoint-meta-telemetry', + passed: true, + }, ], - diagnostics: [ - [ - { - name: 'security:endpoint-diagnostics', - passed: true, - }, - ], + ]); + + expect(stats.diagnostics).to.eql([ + [ + { + name: 'security:endpoint-diagnostics', + passed: true, + }, ], - }); + ]); + + expect(stats.indices_metadata).to.be.an('array'); + const events = stats.indices_metadata as any[]; + + expect(events).to.not.be.empty(); + + const eventTypes = events.map((e) => e.eventType); + expect(eventTypes).to.contain('telemetry_index_stats_event'); + expect(eventTypes).to.contain('telemetry_data_stream_event'); + + const indicesStats = events.find((e) => e.eventType === 'telemetry_index_stats_event'); + expect(indicesStats).to.be.ok(); + expect(indicesStats.eventData).to.be.ok(); + expect(indicesStats.eventData.items).to.not.be.empty(); + expect(indicesStats.eventData.items[0]).to.have.keys( + 'index_name', + 'query_total', + 'query_time_in_millis', + 'docs_count', + 'docs_deleted', + 'docs_total_size_in_bytes' + ); + + const dataStreamStats = events.find((e) => e.eventType === 'telemetry_data_stream_event'); + expect(dataStreamStats).to.be.ok(); + expect(dataStreamStats.eventData).to.be.ok(); + expect(dataStreamStats.eventData.items).to.not.be.empty(); + expect(dataStreamStats.eventData.items[0]).to.have.keys('datastream_name', 'indices'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts index 2d27b375684ba..b835832fcb550 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts @@ -5,19 +5,29 @@ * 2.0. */ -import { unset } from 'lodash'; - export const removeExtraFieldsFromTelemetryStats = (stats: any) => { - Object.entries(stats).forEach(([, value]: [unknown, any]) => { - value.forEach((entry: any, i: number) => { - entry.forEach((_e: any, j: number) => { - unset(value, `[${i}][${j}].time_executed_in_ms`); - unset(value, `[${i}][${j}].start_time`); - unset(value, `[${i}][${j}].end_time`); - unset(value, `[${i}][${j}].cluster_uuid`); - unset(value, `[${i}][${j}].cluster_name`); - unset(value, `[${i}][${j}].license`); - }); - }); - }); + removeExtraFields(stats, [ + 'time_executed_in_ms', + 'start_time', + 'end_time', + 'cluster_uuid', + 'cluster_name', + 'license', + ]); }; + +function removeExtraFields(obj: any, fields: string[]): void { + function traverseAndRemove(o: any): void { + if (typeof o !== 'object' || o === null) return; + + for (const key in o) { + if (fields.includes(key)) { + delete o[key]; + } else if (typeof o[key] === 'object') { + traverseAndRemove(o[key]); + } + } + } + + traverseAndRemove(obj); +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts new file mode 100644 index 0000000000000..5a4bfb39f41f0 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../config/ess/config.base.basic') + ); + + return { + ...functionalConfig.getAll(), + uiSettings: {}, + testFiles: [require.resolve('..')], + junit: { + reportName: 'Security Solution - Telemetry Integration Tests - ESS Env', + }, + }; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts new file mode 100644 index 0000000000000..5c47f36439393 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../config/serverless/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + suiteTags: { exclude: ['skipServerless'] }, + junit: { + reportName: 'Security Solution - Telemetry Integration Tests - Serverless Env', + }, +}); diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts new file mode 100644 index 0000000000000..ff88de12d7124 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default ({ loadTestFile }: FtrProviderContext): void => { + describe('Security Solution - Telemetry', function () { + loadTestFile(require.resolve('./tasks/indices_metadata')); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts new file mode 100644 index 0000000000000..c7d365baab767 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + TELEMETRY_DATA_STREAM_EVENT, + TELEMETRY_ILM_POLICY_EVENT, + TELEMETRY_ILM_STATS_EVENT, + TELEMETRY_INDEX_STATS_EVENT, +} from '@kbn/security-solution-plugin/server/lib/telemetry/event_based/events'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { + cleanupDatastreams, + cleanupPolicies, + ensureBackingIndices, + launchTask, + randomDatastream, + randomIlmPolicy, + taskHasRun, + waitFor, +} from '../../../../common/utils/security_solution'; + +const TASK_ID = 'security:indices-metadata-telemetry:1.0.0'; +const NUM_INDICES = 5; + +export default ({ getService }: FtrProviderContext) => { + const ebtServer = getService('kibana_ebt_server'); + const kibanaServer = getService('kibanaServer'); + const logger = getService('log'); + const es = getService('es'); + + describe('Indices metadata task telemetry', function () { + let dsName: string; + let policyName: string; + + describe('@ess @serverless indices metadata', () => { + beforeEach(async () => { + dsName = await randomDatastream(es); + await ensureBackingIndices(dsName, NUM_INDICES, es); + }); + + afterEach(async () => { + await cleanupDatastreams(es); + }); + + it('should publish data stream events', async () => { + const runAt = await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_DATA_STREAM_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + }; + + await waitFor( + async () => { + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => result.filter((ev) => (ev as any).datastream_name === dsName)); + + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + const eventCount = events.length; + + return hasRun && eventCount === 1; + }, + 'waitForTaskToRun', + logger + ); + }); + + it('should publish index stats events', async () => { + const runAt = await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_INDEX_STATS_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + }; + + // .ds--YYYY.MM.DD-NNNNNN + const regex = new RegExp(`^\.ds-${dsName}-\\d{4}.\\d{2}.\\d{2}-\\d{6}$`); + await waitFor( + async () => { + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => + result.filter((ev) => regex.test((ev as any).index_name as string)) + ); + + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && events.length === NUM_INDICES; + }, + 'waitForTaskToRun', + logger + ); + }); + }); + + describe('@ess indices metadata', function () { + this.tags('skipServerless'); + + beforeEach(async () => { + policyName = await randomIlmPolicy(es); + dsName = await randomDatastream(es, policyName); + await ensureBackingIndices(dsName, NUM_INDICES, es); + }); + + afterEach(async () => { + await cleanupDatastreams(es); + await cleanupPolicies(es); + }); + + it('should publish ilm policy events', async () => { + const runAt = await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_ILM_POLICY_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + }; + + await waitFor( + async () => { + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => result.filter((ev) => (ev as any).policy_name === policyName)); + + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && events.length === 1; + }, + 'waitForTaskToRun', + logger + ); + }); + + it('should publish ilm stats events', async () => { + const runAt = await launchTask(TASK_ID, kibanaServer, logger); + + const opts = { + eventTypes: [TELEMETRY_ILM_STATS_EVENT.eventType], + withTimeoutMs: 1000, + fromTimestamp: new Date().toISOString(), + }; + + await waitFor( + async () => { + const events = await ebtServer + .getEvents(Number.MAX_SAFE_INTEGER, opts) + .then((result) => result.map((ev) => ev.properties.items)) + .then((result) => result.flat()) + .then((result) => result.filter((ev) => (ev as any).policy_name === policyName)); + + const hasRun = await taskHasRun(TASK_ID, kibanaServer, runAt); + + return hasRun && events.length === NUM_INDICES; + }, + 'waitForTaskToRun', + logger + ); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/tsconfig.json b/x-pack/test/security_solution_api_integration/tsconfig.json index e13f4bd61520d..26f1ee30916dc 100644 --- a/x-pack/test/security_solution_api_integration/tsconfig.json +++ b/x-pack/test/security_solution_api_integration/tsconfig.json @@ -52,5 +52,6 @@ "@kbn/ftr-common-functional-ui-services", "@kbn/spaces-plugin", "@kbn/elastic-assistant-plugin", + "@kbn/test-suites-src", ] }