[Telemetry][Security Solution] Index metadata collector (elastic#194004)

## Summary

Implements a security_solution task scheduled to run once a day to
collect the following information:

1. Datastreams stats
2. Indices stats
3. ILMs stats
4. ILM configs

The task allows a runtime configuration to limit the number of indices
and data streams to analyze or event to disable the feature entirely.

Once the data is gathered, the task sends it as EBT events.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.buildkite/ftr_security_serverless_configs.yml

@@ -89,6 +89,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -70,6 +70,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts

x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts

@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import Path from 'path';
+import axios from 'axios';
+
+import { cloneDeep } from 'lodash';
+
+import { telemetryConfiguration } from '../lib/telemetry/configuration';
+import {
+  TaskManagerPlugin,
+  type TaskManagerStartContract,
+} from '@kbn/task-manager-plugin/server/plugin';
+
+import {
+  setupTestServers,
+  removeFile,
+  mockAxiosPost,
+  DEFAULT_GET_ROUTES,
+  mockAxiosGet,
+  getRandomInt,
+} from './lib/helpers';
+
+import {
+  type TestElasticsearchUtils,
+  type TestKibanaUtils,
+} from '@kbn/core-test-helpers-kbn-server';
+import { Plugin as SecuritySolutionPlugin } from '../plugin';
+import { getTelemetryTasks, runSoonConfigTask } from './lib/telemetry_helpers';
+import type { SecurityTelemetryTask } from '../lib/telemetry/task';
+
+jest.mock('axios');
+
+const logFilePath = Path.join(__dirname, 'config.logs.log');
+const taskManagerStartSpy = jest.spyOn(TaskManagerPlugin.prototype, 'start');
+const securitySolutionStartSpy = jest.spyOn(SecuritySolutionPlugin.prototype, 'start');
+
+const mockedAxiosGet = jest.spyOn(axios, 'get');
+const mockedAxiosPost = jest.spyOn(axios, 'post');
+
+const securitySolutionPlugin = jest.spyOn(SecuritySolutionPlugin.prototype, 'start');
+
+describe('configuration', () => {
+  let esServer: TestElasticsearchUtils;
+  let kibanaServer: TestKibanaUtils;
+  let taskManagerPlugin: TaskManagerStartContract;
+  let tasks: SecurityTelemetryTask[];
+
+  beforeAll(async () => {
+    await removeFile(logFilePath);
+
+    const servers = await setupTestServers(logFilePath);
+
+    esServer = servers.esServer;
+    kibanaServer = servers.kibanaServer;
+
+    expect(taskManagerStartSpy).toHaveBeenCalledTimes(1);
+    taskManagerPlugin = taskManagerStartSpy.mock.results[0].value;
+
+    expect(securitySolutionStartSpy).toHaveBeenCalledTimes(1);
+
+    tasks = getTelemetryTasks(securitySolutionStartSpy);
+
+    expect(securitySolutionPlugin).toHaveBeenCalledTimes(1);
+  });
+
+  afterAll(async () => {
+    if (kibanaServer) {
+      await kibanaServer.stop();
+    }
+    if (esServer) {
+      await esServer.stop();
+    }
+  });
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+    mockAxiosPost(mockedAxiosPost);
+  });
+
+  afterEach(async () => {});
+
+  describe('configuration task', () => {
+    it('should keep default values when no new config was provided', async () => {
+      const before = cloneDeep(telemetryConfiguration);
+
+      await runSoonConfigTask(tasks, taskManagerPlugin);
+
+      expect(telemetryConfiguration).toEqual(before);
+    });
+
+    it('should update values with new manifest', async () => {
+      const expected = {
+        telemetry_max_buffer_size: getRandomInt(1, 100),
+        max_security_list_telemetry_batch: getRandomInt(1, 100),
+        max_endpoint_telemetry_batch: getRandomInt(1, 100),
+        max_detection_rule_telemetry_batch: getRandomInt(1, 100),
+        max_detection_alerts_batch: getRandomInt(1, 100),
+        use_async_sender: true,
+        pagination_config: {
+          max_page_size_bytes: getRandomInt(1, 100),
+          num_docs_to_sample: getRandomInt(1, 100),
+        },
+        sender_channels: {
+          default: {
+            buffer_time_span_millis: getRandomInt(1, 100),
+            inflight_events_threshold: getRandomInt(1, 100),
+            max_payload_size_bytes: getRandomInt(1, 100),
+          },
+        },
+        indices_metadata_config: {
+          indices_threshold: getRandomInt(1, 100),
+          datastreams_threshold: getRandomInt(1, 100),
+          max_prefixes: getRandomInt(1, 100),
+          max_group_size: getRandomInt(1, 100),
+        },
+      };
+
+      mockAxiosGet(mockedAxiosGet, [
+        ...DEFAULT_GET_ROUTES,
+        [/.*telemetry-buffer-and-batch-sizes-v1.*/, { status: 200, data: cloneDeep(expected) }],
+      ]);
+
+      await runSoonConfigTask(tasks, taskManagerPlugin);
+
+      expect(telemetryConfiguration.telemetry_max_buffer_size).toEqual(
+        expected.telemetry_max_buffer_size
+      );
+      expect(telemetryConfiguration.max_security_list_telemetry_batch).toEqual(
+        expected.max_security_list_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_endpoint_telemetry_batch).toEqual(
+        expected.max_endpoint_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_detection_rule_telemetry_batch).toEqual(
+        expected.max_detection_rule_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_detection_alerts_batch).toEqual(
+        expected.max_detection_alerts_batch
+      );
+      expect(telemetryConfiguration.use_async_sender).toEqual(expected.use_async_sender);
+      expect(telemetryConfiguration.sender_channels).toEqual(expected.sender_channels);
+      expect(telemetryConfiguration.pagination_config).toEqual(expected.pagination_config);
+      expect(telemetryConfiguration.indices_metadata_config).toEqual(
+        expected.indices_metadata_config
+      );
+    });
+  });
+});

x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts

@@ -10,6 +10,20 @@ import Util from 'util';
 import type { ElasticsearchClient } from '@kbn/core/server';
 import deepmerge from 'deepmerge';
 import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
+
+export const DEFAULT_GET_ROUTES: Array<[RegExp, unknown]> = [
+  [new RegExp('.*/ping$'), { status: 200 }],
+  [
+    /.*kibana\/manifest\/artifacts.*/,
+    {
+      status: 200,
+      data: 'x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/kibana-artifacts.zip',
+    },
+  ],
+];
+
+export const DEFAULT_POST_ROUTES: Array<[RegExp, unknown]> = [[/.*/, { status: 200 }]];
+
 const asyncUnlink = Util.promisify(Fs.unlink);
 
 /**
@@ -127,3 +141,35 @@ export function updateTimestamps(data: object[]): object[] {
     return { ...d, '@timestamp': new Date(currentTimeMillis + (i + 1) * 100) };
   });
 }
+
+export function mockAxiosPost(
+  postSpy: jest.SpyInstance,
+  routes: Array<[RegExp, unknown]> = DEFAULT_POST_ROUTES
+) {
+  postSpy.mockImplementation(async (url: string) => {
+    for (const [route, returnValue] of routes) {
+      if (route.test(url)) {
+        return returnValue;
+      }
+    }
+    return { status: 404 };
+  });
+}
+
+export function mockAxiosGet(
+  getSpy: jest.SpyInstance,
+  routes: Array<[RegExp, unknown]> = DEFAULT_GET_ROUTES
+) {
+  getSpy.mockImplementation(async (url: string) => {
+    for (const [route, returnValue] of routes) {
+      if (route.test(url)) {
+        return returnValue;
+      }
+    }
+    return { status: 404 };
+  });
+}
+
+export function getRandomInt(min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min + 1)) + min;
+}

x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts

@@ -25,13 +25,13 @@ import {
   deleteExceptionListItem,
 } from '@kbn/lists-plugin/server/services/exception_lists';
 import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants';
+import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
 
 import { packagePolicyService } from '@kbn/fleet-plugin/server/services';
 
 import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
 import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock';
-import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants';
-import { bulkInsert, updateTimestamps } from './helpers';
+import { bulkInsert, eventually, updateTimestamps } from './helpers';
 import { TelemetryEventsSender } from '../../lib/telemetry/sender';
 import type {
   SecuritySolutionPluginStart,
@@ -41,6 +41,7 @@ import type { SecurityTelemetryTask } from '../../lib/telemetry/task';
 import { Plugin as SecuritySolutionPlugin } from '../../plugin';
 import { AsyncTelemetryEventsSender } from '../../lib/telemetry/async_sender';
 import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/receiver';
+import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants';
 import mockEndpointAlert from '../__mocks__/endpoint-alert.json';
 import mockedRule from '../__mocks__/rule.json';
 import fleetAgents from '../__mocks__/fleet-agents.json';
@@ -417,3 +418,24 @@ export function getTelemetryTaskType(task: SecurityTelemetryTask): string {
     return '';
   }
 }
+
+export async function runSoonConfigTask(
+  tasks: SecurityTelemetryTask[],
+  taskManagerPlugin: TaskManagerStartContract
+) {
+  const configTaskType = 'security:telemetry-configuration';
+  const configTask = getTelemetryTask(tasks, configTaskType);
+  const runAfter = new Date();
+  await eventually(async () => {
+    await taskManagerPlugin.runSoon(configTask.getTaskId());
+  });
+
+  // wait until the task finishes
+  await eventually(async () => {
+    const hasRun = await taskManagerPlugin
+      .get(configTask.getTaskId())
+      .then((t) => new Date(t.state.lastExecutionTimestamp) > runAfter)
+      .catch(() => false);
+    expect(hasRun).toBe(true);
+  });
+}