Skip to content

Commit

Permalink
[Security Solution] Move ES|QL parsing functionality into `@kbn/secur…
Browse files Browse the repository at this point in the history
…itysolution-utils` package (elastic#202772)

## Summary

With this PR we move existing `parseEsqlQuery` method into a shared
security solution utils package. We need to the same functionality in
"SIEM migrations" feature. Previously we duplicated the code in [this
PR](https://github.com/elastic/kibana/pull/202331/files#diff-b5f1a952a5e5a9685a4fef5d1f5a4c3b53ce338333e569bb6f92ccf2681100b7R54)
and these are the follow-up changes to make parsing functionality shared
for easier re-use within security solution.

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
2 people authored and CAWilson94 committed Dec 9, 2024
1 parent 28a0500 commit cd30c65
Show file tree
Hide file tree
Showing 7 changed files with 150 additions and 78 deletions.
1 change: 1 addition & 0 deletions packages/kbn-securitysolution-utils/src/esql/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@

export * from './compute_if_esql_query_aggregating';
export * from './get_index_list_from_esql_query';
export * from './parse_esql_query';
119 changes: 119 additions & 0 deletions packages/kbn-securitysolution-utils/src/esql/parse_esql_query.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { parseEsqlQuery } from './parse_esql_query';

describe('parseEsqlQuery', () => {
describe('ES|QL query syntax', () => {
it.each([['incorrect syntax'], ['from test* metadata']])(
'detects incorrect syntax in "%s"',
(esqlQuery) => {
const result = parseEsqlQuery(esqlQuery);
expect(result.errors.length).toEqual(1);
expect(result.errors[0].message.startsWith('SyntaxError:')).toBeTruthy();
expect(parseEsqlQuery(esqlQuery)).toMatchObject({
hasMetadataOperator: false,
isEsqlQueryAggregating: false,
});
}
);

it.each([
['from test* metadata _id'],
[
'FROM kibana_sample_data_logs | STATS total_bytes = SUM(bytes) BY host | WHERE total_bytes > 200000 | SORT total_bytes DESC | LIMIT 10',
],
[
`from packetbeat* metadata
_id
| limit 100`,
],
[
`FROM kibana_sample_data_logs |
STATS total_bytes = SUM(bytes) BY host |
WHERE total_bytes > 200000 |
SORT total_bytes DESC |
LIMIT 10`,
],
])('parses correctly valid syntax in "%s"', (esqlQuery) => {
const result = parseEsqlQuery(esqlQuery);
expect(result.errors.length).toEqual(0);
expect(result).toMatchObject({ errors: [] });
});
});

describe('METADATA operator', () => {
it.each([
['from test*'],
['from metadata*'],
['from test* | keep metadata'],
['from test* | eval x="metadata _id"'],
])('detects when METADATA operator is missing in a NON aggregating query "%s"', (esqlQuery) => {
expect(parseEsqlQuery(esqlQuery)).toEqual({
errors: [],
hasMetadataOperator: false,
isEsqlQueryAggregating: false,
});
});

it.each([
['from test* metadata _id'],
['from test* metadata _id, _index'],
['from test* metadata _index, _id'],
['from test* metadata _id '],
['from test* metadata _id '],
['from test* metadata _id | limit 10'],
[
`from packetbeat* metadata
_id
| limit 100`,
],
])('detects existin METADATA operator in a NON aggregating query "%s"', (esqlQuery) =>
expect(parseEsqlQuery(esqlQuery)).toEqual({
errors: [],
hasMetadataOperator: true,
isEsqlQueryAggregating: false,
})
);

it('detects missing METADATA operator in an aggregating query "from test* | stats c = count(*) by fieldA"', () =>
expect(parseEsqlQuery('from test* | stats c = count(*) by fieldA')).toEqual({
errors: [],
hasMetadataOperator: false,
isEsqlQueryAggregating: true,
}));
});

describe('METADATA _id field for NON aggregating queries', () => {
it('detects missing METADATA "_id" field', () => {
expect(parseEsqlQuery('from test*')).toEqual({
errors: [],
hasMetadataOperator: false,
isEsqlQueryAggregating: false,
});
});

it('detects existing METADATA "_id" field', async () => {
expect(parseEsqlQuery('from test* metadata _id')).toEqual({
errors: [],
hasMetadataOperator: true,
isEsqlQueryAggregating: false,
});
});
});

describe('METADATA _id field for aggregating queries', () => {
it('detects existing METADATA operator with missing "_id" field', () => {
expect(
parseEsqlQuery('from test* metadata someField | stats c = count(*) by fieldA')
).toEqual({ errors: [], hasMetadataOperator: false, isEsqlQueryAggregating: true });
});
});
});
Original file line number Diff line number Diff line change
@@ -1,21 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import type { ESQLAstQueryExpression, ESQLCommandOption, EditorError } from '@kbn/esql-ast';
import { parse } from '@kbn/esql-ast';
import { type ESQLAstQueryExpression, parse, ESQLCommandOption, EditorError } from '@kbn/esql-ast';
import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete';
import { isAggregatingQuery } from '@kbn/securitysolution-utils';
import { isAggregatingQuery } from './compute_if_esql_query_aggregating';

interface ParseEsqlQueryResult {
export interface ParseEsqlQueryResult {
errors: EditorError[];
isEsqlQueryAggregating: boolean;
hasMetadataOperator: boolean;
}

/**
* check if esql query valid for Security rule:
* - if it's non aggregation query it must have metadata operator
*/
export const parseEsqlQuery = (query: string): ParseEsqlQueryResult => {
const { root, errors } = parse(query);
const isEsqlQueryAggregating = isAggregatingQuery(root);

return {
errors,
isEsqlQueryAggregating,
hasMetadataOperator: computeHasMetadataOperator(root),
};
};

/**
* checks whether query has metadata _id operator
*/
function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean {
// Check whether the `from` command has `metadata` operator
const metadataOption = getMetadataOption(astExpression);
Expand Down Expand Up @@ -50,13 +69,3 @@ function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOp

return undefined;
}

export const parseEsqlQuery = (query: string): ParseEsqlQueryResult => {
const { root, errors } = parse(query);
const isEsqlQueryAggregating = isAggregatingQuery(root);
return {
errors,
isEsqlQueryAggregating,
hasMetadataOperator: computeHasMetadataOperator(root),
};
};
3 changes: 2 additions & 1 deletion packages/kbn-securitysolution-utils/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
"kbn_references": [
"@kbn/i18n",
"@kbn/esql-utils",
"@kbn/esql-ast"
"@kbn/esql-ast",
"@kbn/esql-validation-autocomplete"
],
"exclude": [
"target/**/*",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@

import type { QueryClient } from '@tanstack/react-query';
import type { DatatableColumn } from '@kbn/expressions-plugin/common';
import type { ESQLAstQueryExpression, ESQLCommandOption } from '@kbn/esql-ast';
import { parse } from '@kbn/esql-ast';
import { isAggregatingQuery } from '@kbn/securitysolution-utils';
import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete';
import { parseEsqlQuery } from '@kbn/securitysolution-utils';
import type { FormData, ValidationError, ValidationFunc } from '../../../../../shared_imports';
import type { FieldValueQueryBar } from '../../../../rule_creation_ui/components/query_bar_field';
import { fetchEsqlQueryColumns } from '../../../logic/esql_query_columns';
Expand Down Expand Up @@ -79,59 +76,6 @@ function hasIdColumn(columns: DatatableColumn[]): boolean {
return columns.some(({ id }) => '_id' === id);
}

/**
* check if esql query valid for Security rule:
* - if it's non aggregation query it must have metadata operator
*/
function parseEsqlQuery(query: string) {
const { root, errors } = parse(query);
const isEsqlQueryAggregating = isAggregatingQuery(root);

return {
errors,
isEsqlQueryAggregating,
hasMetadataOperator: computeHasMetadataOperator(root),
};
}

/**
* checks whether query has metadata _id operator
*/
function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean {
// Check whether the `from` command has `metadata` operator
const metadataOption = getMetadataOption(astExpression);
if (!metadataOption) {
return false;
}

// Check whether the `metadata` operator has `_id` argument
const idColumnItem = metadataOption.args.find(
(fromArg) => isColumnItem(fromArg) && fromArg.name === '_id'
);
if (!idColumnItem) {
return false;
}

return true;
}

function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOption | undefined {
const fromCommand = astExpression.commands.find((x) => x.name === 'from');

if (!fromCommand?.args) {
return undefined;
}

// Check whether the `from` command has `metadata` operator
for (const fromArg of fromCommand.args) {
if (isOptionItem(fromArg) && fromArg.name === 'metadata') {
return fromArg;
}
}

return undefined;
}

function constructSyntaxError(error: Error): ValidationError {
return {
code: ESQL_ERROR_CODES.INVALID_SYNTAX,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@

import type { Logger } from '@kbn/core/server';
import { isEmpty } from 'lodash/fp';
import { parseEsqlQuery } from '@kbn/securitysolution-utils';
import type { GraphNode } from '../../types';
import { parseEsqlQuery } from './esql_query';

interface GetValidationNodeParams {
logger: Logger;
Expand Down
2 changes: 0 additions & 2 deletions x-pack/plugins/security_solution/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,6 @@
"@kbn/core-theme-browser",
"@kbn/integration-assistant-plugin",
"@kbn/avc-banner",
"@kbn/esql-ast",
"@kbn/esql-validation-autocomplete",
"@kbn/config",
"@kbn/openapi-common",
"@kbn/securitysolution-lists-common",
Expand Down

0 comments on commit cd30c65

Please sign in to comment.