From ff57bf3d02f68ab9f199ac09af5760bcab7d02f6 Mon Sep 17 00:00:00 2001 From: Ievgen Sorokopud Date: Thu, 5 Dec 2024 10:12:32 +0100 Subject: [PATCH] [Security Solution] Move ES|QL parsing functionality into `@kbn/securitysolution-utils` package (#202772) ## Summary With this PR we move existing `parseEsqlQuery` method into a shared security solution utils package. We need to the same functionality in "SIEM migrations" feature. Previously we duplicated the code in [this PR](https://github.com/elastic/kibana/pull/202331/files#diff-b5f1a952a5e5a9685a4fef5d1f5a4c3b53ce338333e569bb6f92ccf2681100b7R54) and these are the follow-up changes to make parsing functionality shared for easier re-use within security solution. ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../src/esql/index.ts | 1 + .../src/esql/parse_esql_query.test.ts | 119 ++++++++++++++++++ .../src/esql/parse_esql_query.ts | 43 ++++--- .../kbn-securitysolution-utils/tsconfig.json | 3 +- .../esql_query_validator_factory.ts | 58 +-------- .../nodes/validation/validation.ts | 2 +- .../plugins/security_solution/tsconfig.json | 2 - 7 files changed, 150 insertions(+), 78 deletions(-) create mode 100644 packages/kbn-securitysolution-utils/src/esql/parse_esql_query.test.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/esql_query.ts => packages/kbn-securitysolution-utils/src/esql/parse_esql_query.ts (66%) diff --git a/packages/kbn-securitysolution-utils/src/esql/index.ts b/packages/kbn-securitysolution-utils/src/esql/index.ts index 22c2cc42a9fed..930ff246988ea 100644 --- a/packages/kbn-securitysolution-utils/src/esql/index.ts +++ b/packages/kbn-securitysolution-utils/src/esql/index.ts @@ -9,3 +9,4 @@ export * from './compute_if_esql_query_aggregating'; export * from './get_index_list_from_esql_query'; +export * from './parse_esql_query'; diff --git a/packages/kbn-securitysolution-utils/src/esql/parse_esql_query.test.ts b/packages/kbn-securitysolution-utils/src/esql/parse_esql_query.test.ts new file mode 100644 index 0000000000000..6c4fdafd8e70b --- /dev/null +++ b/packages/kbn-securitysolution-utils/src/esql/parse_esql_query.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { parseEsqlQuery } from './parse_esql_query'; + +describe('parseEsqlQuery', () => { + describe('ES|QL query syntax', () => { + it.each([['incorrect syntax'], ['from test* metadata']])( + 'detects incorrect syntax in "%s"', + (esqlQuery) => { + const result = parseEsqlQuery(esqlQuery); + expect(result.errors.length).toEqual(1); + expect(result.errors[0].message.startsWith('SyntaxError:')).toBeTruthy(); + expect(parseEsqlQuery(esqlQuery)).toMatchObject({ + hasMetadataOperator: false, + isEsqlQueryAggregating: false, + }); + } + ); + + it.each([ + ['from test* metadata _id'], + [ + 'FROM kibana_sample_data_logs | STATS total_bytes = SUM(bytes) BY host | WHERE total_bytes > 200000 | SORT total_bytes DESC | LIMIT 10', + ], + [ + `from packetbeat* metadata + _id + | limit 100`, + ], + [ + `FROM kibana_sample_data_logs | + STATS total_bytes = SUM(bytes) BY host | + WHERE total_bytes > 200000 | + SORT total_bytes DESC | + LIMIT 10`, + ], + ])('parses correctly valid syntax in "%s"', (esqlQuery) => { + const result = parseEsqlQuery(esqlQuery); + expect(result.errors.length).toEqual(0); + expect(result).toMatchObject({ errors: [] }); + }); + }); + + describe('METADATA operator', () => { + it.each([ + ['from test*'], + ['from metadata*'], + ['from test* | keep metadata'], + ['from test* | eval x="metadata _id"'], + ])('detects when METADATA operator is missing in a NON aggregating query "%s"', (esqlQuery) => { + expect(parseEsqlQuery(esqlQuery)).toEqual({ + errors: [], + hasMetadataOperator: false, + isEsqlQueryAggregating: false, + }); + }); + + it.each([ + ['from test* metadata _id'], + ['from test* metadata _id, _index'], + ['from test* metadata _index, _id'], + ['from test* metadata _id '], + ['from test* metadata _id '], + ['from test* metadata _id | limit 10'], + [ + `from packetbeat* metadata + + _id + | limit 100`, + ], + ])('detects existin METADATA operator in a NON aggregating query "%s"', (esqlQuery) => + expect(parseEsqlQuery(esqlQuery)).toEqual({ + errors: [], + hasMetadataOperator: true, + isEsqlQueryAggregating: false, + }) + ); + + it('detects missing METADATA operator in an aggregating query "from test* | stats c = count(*) by fieldA"', () => + expect(parseEsqlQuery('from test* | stats c = count(*) by fieldA')).toEqual({ + errors: [], + hasMetadataOperator: false, + isEsqlQueryAggregating: true, + })); + }); + + describe('METADATA _id field for NON aggregating queries', () => { + it('detects missing METADATA "_id" field', () => { + expect(parseEsqlQuery('from test*')).toEqual({ + errors: [], + hasMetadataOperator: false, + isEsqlQueryAggregating: false, + }); + }); + + it('detects existing METADATA "_id" field', async () => { + expect(parseEsqlQuery('from test* metadata _id')).toEqual({ + errors: [], + hasMetadataOperator: true, + isEsqlQueryAggregating: false, + }); + }); + }); + + describe('METADATA _id field for aggregating queries', () => { + it('detects existing METADATA operator with missing "_id" field', () => { + expect( + parseEsqlQuery('from test* metadata someField | stats c = count(*) by fieldA') + ).toEqual({ errors: [], hasMetadataOperator: false, isEsqlQueryAggregating: true }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/esql_query.ts b/packages/kbn-securitysolution-utils/src/esql/parse_esql_query.ts similarity index 66% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/esql_query.ts rename to packages/kbn-securitysolution-utils/src/esql/parse_esql_query.ts index a8c1d6acff408..2a62aed8873a0 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/esql_query.ts +++ b/packages/kbn-securitysolution-utils/src/esql/parse_esql_query.ts @@ -1,21 +1,40 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -import type { ESQLAstQueryExpression, ESQLCommandOption, EditorError } from '@kbn/esql-ast'; -import { parse } from '@kbn/esql-ast'; +import { type ESQLAstQueryExpression, parse, ESQLCommandOption, EditorError } from '@kbn/esql-ast'; import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete'; -import { isAggregatingQuery } from '@kbn/securitysolution-utils'; +import { isAggregatingQuery } from './compute_if_esql_query_aggregating'; -interface ParseEsqlQueryResult { +export interface ParseEsqlQueryResult { errors: EditorError[]; isEsqlQueryAggregating: boolean; hasMetadataOperator: boolean; } +/** + * check if esql query valid for Security rule: + * - if it's non aggregation query it must have metadata operator + */ +export const parseEsqlQuery = (query: string): ParseEsqlQueryResult => { + const { root, errors } = parse(query); + const isEsqlQueryAggregating = isAggregatingQuery(root); + + return { + errors, + isEsqlQueryAggregating, + hasMetadataOperator: computeHasMetadataOperator(root), + }; +}; + +/** + * checks whether query has metadata _id operator + */ function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean { // Check whether the `from` command has `metadata` operator const metadataOption = getMetadataOption(astExpression); @@ -50,13 +69,3 @@ function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOp return undefined; } - -export const parseEsqlQuery = (query: string): ParseEsqlQueryResult => { - const { root, errors } = parse(query); - const isEsqlQueryAggregating = isAggregatingQuery(root); - return { - errors, - isEsqlQueryAggregating, - hasMetadataOperator: computeHasMetadataOperator(root), - }; -}; diff --git a/packages/kbn-securitysolution-utils/tsconfig.json b/packages/kbn-securitysolution-utils/tsconfig.json index 5b9520c487e31..d45b0c973af87 100644 --- a/packages/kbn-securitysolution-utils/tsconfig.json +++ b/packages/kbn-securitysolution-utils/tsconfig.json @@ -13,7 +13,8 @@ "kbn_references": [ "@kbn/i18n", "@kbn/esql-utils", - "@kbn/esql-ast" + "@kbn/esql-ast", + "@kbn/esql-validation-autocomplete" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts index 90cdaff14cc9b..c5b54db172a18 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts @@ -7,10 +7,7 @@ import type { QueryClient } from '@tanstack/react-query'; import type { DatatableColumn } from '@kbn/expressions-plugin/common'; -import type { ESQLAstQueryExpression, ESQLCommandOption } from '@kbn/esql-ast'; -import { parse } from '@kbn/esql-ast'; -import { isAggregatingQuery } from '@kbn/securitysolution-utils'; -import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete'; +import { parseEsqlQuery } from '@kbn/securitysolution-utils'; import type { FormData, ValidationError, ValidationFunc } from '../../../../../shared_imports'; import type { FieldValueQueryBar } from '../../../../rule_creation_ui/components/query_bar_field'; import { fetchEsqlQueryColumns } from '../../../logic/esql_query_columns'; @@ -79,59 +76,6 @@ function hasIdColumn(columns: DatatableColumn[]): boolean { return columns.some(({ id }) => '_id' === id); } -/** - * check if esql query valid for Security rule: - * - if it's non aggregation query it must have metadata operator - */ -function parseEsqlQuery(query: string) { - const { root, errors } = parse(query); - const isEsqlQueryAggregating = isAggregatingQuery(root); - - return { - errors, - isEsqlQueryAggregating, - hasMetadataOperator: computeHasMetadataOperator(root), - }; -} - -/** - * checks whether query has metadata _id operator - */ -function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean { - // Check whether the `from` command has `metadata` operator - const metadataOption = getMetadataOption(astExpression); - if (!metadataOption) { - return false; - } - - // Check whether the `metadata` operator has `_id` argument - const idColumnItem = metadataOption.args.find( - (fromArg) => isColumnItem(fromArg) && fromArg.name === '_id' - ); - if (!idColumnItem) { - return false; - } - - return true; -} - -function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOption | undefined { - const fromCommand = astExpression.commands.find((x) => x.name === 'from'); - - if (!fromCommand?.args) { - return undefined; - } - - // Check whether the `from` command has `metadata` operator - for (const fromArg of fromCommand.args) { - if (isOptionItem(fromArg) && fromArg.name === 'metadata') { - return fromArg; - } - } - - return undefined; -} - function constructSyntaxError(error: Error): ValidationError { return { code: ESQL_ERROR_CODES.INVALID_SYNTAX, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts index 272aedfe4793d..6f97678a04558 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts @@ -7,8 +7,8 @@ import type { Logger } from '@kbn/core/server'; import { isEmpty } from 'lodash/fp'; +import { parseEsqlQuery } from '@kbn/securitysolution-utils'; import type { GraphNode } from '../../types'; -import { parseEsqlQuery } from './esql_query'; interface GetValidationNodeParams { logger: Logger; diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index 4d11804796e2b..7767fffe69824 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -209,8 +209,6 @@ "@kbn/core-theme-browser", "@kbn/integration-assistant-plugin", "@kbn/avc-banner", - "@kbn/esql-ast", - "@kbn/esql-validation-autocomplete", "@kbn/config", "@kbn/openapi-common", "@kbn/securitysolution-lists-common",