From bec0a8aa1d4423aea360b5141fbe26e065e7f0b3 Mon Sep 17 00:00:00 2001 From: Alex Tarasov Date: Sat, 27 Jul 2024 17:38:24 +0100 Subject: [PATCH] fix: opensearch api as separate construct --- framework/API.md | 390 ++++++++++++++++++ .../examples/opensearch-api.lit.ts | 12 +- .../examples/opensearch-saml-clientvpn.lit.ts | 4 +- .../examples/opensearch-saml.lit.ts | 4 +- .../src/consumption/lib/opensearch/index.ts | 2 + .../lib/opensearch/opensearch-api-props.ts | 67 +++ .../lib/opensearch/opensearch-api.ts | 126 ++++++ .../consumption/lib/opensearch/opensearch.ts | 107 ++--- framework/test/e2e/opensearch-api.e2e.test.ts | 71 ++++ .../unit/consumption/opensearch-api.test.ts | 53 +++ .../consumption/nag-opensearch-api.test.ts | 155 +++++++ .../nag/consumption/nag-opensearch.test.ts | 6 +- .../generated/_consumption-opensearch.mdx | 61 +-- 13 files changed, 943 insertions(+), 115 deletions(-) create mode 100644 framework/src/consumption/lib/opensearch/opensearch-api-props.ts create mode 100644 framework/src/consumption/lib/opensearch/opensearch-api.ts create mode 100644 framework/test/e2e/opensearch-api.e2e.test.ts create mode 100644 framework/test/unit/consumption/opensearch-api.test.ts create mode 100644 framework/test/unit/nag/consumption/nag-opensearch-api.test.ts diff --git a/framework/API.md b/framework/API.md index 79ee1a26e..c81cbcf49 100644 --- a/framework/API.md +++ b/framework/API.md @@ -5708,6 +5708,248 @@ public readonly DSF_TRACKING_CODE: string; --- +### OpenSearchApi + +A construct to create an OpenSearch API client. + +#### Initializers + +```typescript +import { consumption } from '@cdklabs/aws-data-solutions-framework' + +new consumption.OpenSearchApi(scope: Construct, id: string, props: OpenSearchApiProps) +``` + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| scope | constructs.Construct | Construct. | +| id | string | unique ID for the construct. | +| props | @cdklabs/aws-data-solutions-framework.consumption.OpenSearchApiProps | *No description.* | + +--- + +##### `scope`Required + +- *Type:* constructs.Construct + +Construct. + +--- + +##### `id`Required + +- *Type:* string + +unique ID for the construct. + +--- + +##### `props`Required + +- *Type:* @cdklabs/aws-data-solutions-framework.consumption.OpenSearchApiProps + +--- + +#### Methods + +| **Name** | **Description** | +| --- | --- | +| toString | Returns a string representation of this construct. | +| addRoleMapping | *No description.* | +| callOpenSearchApi | Calls OpenSearch API using custom resource. | +| retrieveVersion | Retrieve DSF package.json version. | + +--- + +##### `toString` + +```typescript +public toString(): string +``` + +Returns a string representation of this construct. + +##### `addRoleMapping` + +```typescript +public addRoleMapping(id: string, name: string, role: string, persist?: boolean): CustomResource +``` + +> [https://opensearch.org/docs/2.9/security/access-control/users-roles/#predefined-roles](https://opensearch.org/docs/2.9/security/access-control/users-roles/#predefined-roles) + +###### `id`Required + +- *Type:* string + +The CDK resource ID. + +--- + +###### `name`Required + +- *Type:* string + +OpenSearch role name. + +--- + +###### `role`Required + +- *Type:* string + +list of IAM roles. + +For IAM Identity center provide SAML group Id as a role + +--- + +###### `persist`Optional + +- *Type:* boolean + +Set to true if you want to prevent the roles to be ovewritten by subsequent PUT API calls. + +Default false. + +--- + +##### `callOpenSearchApi` + +```typescript +public callOpenSearchApi(id: string, apiPath: string, body: any, method?: string): CustomResource +``` + +Calls OpenSearch API using custom resource. + +###### `id`Required + +- *Type:* string + +The CDK resource ID. + +--- + +###### `apiPath`Required + +- *Type:* string + +OpenSearch API path. + +--- + +###### `body`Required + +- *Type:* any + +OpenSearch API request body. + +--- + +###### `method`Optional + +- *Type:* string + +Opensearch API method,. + +--- + +##### `retrieveVersion` + +```typescript +public retrieveVersion(): any +``` + +Retrieve DSF package.json version. + +#### Static Functions + +| **Name** | **Description** | +| --- | --- | +| isConstruct | Checks if `x` is a construct. | + +--- + +##### `isConstruct` + +```typescript +import { consumption } from '@cdklabs/aws-data-solutions-framework' + +consumption.OpenSearchApi.isConstruct(x: any) +``` + +Checks if `x` is a construct. + +Use this method instead of `instanceof` to properly detect `Construct` +instances, even when the construct library is symlinked. + +Explanation: in JavaScript, multiple copies of the `constructs` library on +disk are seen as independent, completely different libraries. As a +consequence, the class `Construct` in each copy of the `constructs` library +is seen as a different class, and an instance of one class will not test as +`instanceof` the other class. `npm install` will not create installations +like this, but users may manually symlink construct libraries together or +use a monorepo tool: in those cases, multiple copies of the `constructs` +library can be accidentally installed, and `instanceof` will behave +unpredictably. It is safest to avoid using `instanceof`, and using +this type-testing method instead. + +###### `x`Required + +- *Type:* any + +Any object. + +--- + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| node | constructs.Node | The tree node. | + +--- + +##### `node`Required + +```typescript +public readonly node: Node; +``` + +- *Type:* constructs.Node + +The tree node. + +--- + +#### Constants + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| DSF_OWNED_TAG | string | *No description.* | +| DSF_TRACKING_CODE | string | *No description.* | + +--- + +##### `DSF_OWNED_TAG`Required + +```typescript +public readonly DSF_OWNED_TAG: string; +``` + +- *Type:* string + +--- + +##### `DSF_TRACKING_CODE`Required + +```typescript +public readonly DSF_TRACKING_CODE: string; +``` + +- *Type:* string + +--- + ### OpenSearchCluster A construct to provision Amazon OpenSearch Cluster and OpenSearch Dashboards. @@ -13888,6 +14130,133 @@ This parameter should not be provided for MSK Serverless. --- +### OpenSearchApiProps + +Configuration for the OpenSearch API. + +#### Initializer + +```typescript +import { consumption } from '@cdklabs/aws-data-solutions-framework' + +const openSearchApiProps: consumption.OpenSearchApiProps = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| openSearchClusterType | @cdklabs/aws-data-solutions-framework.consumption.OpenSearchClusterType | Type of OpenSearch cluster. | +| openSearchEndpoint | string | The OpenSearch Cluster or Serverless collection endpoint to connect to. | +| iamHandlerRole | aws-cdk-lib.aws_iam.IRole | The IAM role to pass to IAM authentication lambda handler This role must be able to be assumed with `lambda.amazonaws.com` service principal. | +| openSearchEndpointRegion | string | AWS Region openSearchEndpoint is provisioned in. | +| removalPolicy | aws-cdk-lib.RemovalPolicy | The removal policy when deleting the CDK resource. | +| subnets | aws-cdk-lib.aws_ec2.SubnetSelection | The subnets where the Custom Resource Lambda Function would be created in. | +| vpc | aws-cdk-lib.aws_ec2.IVpc | Defines the virtual networking environment for this construct. | + +--- + +##### `openSearchClusterType`Required + +```typescript +public readonly openSearchClusterType: OpenSearchClusterType; +``` + +- *Type:* @cdklabs/aws-data-solutions-framework.consumption.OpenSearchClusterType + +Type of OpenSearch cluster. + +--- + +##### `openSearchEndpoint`Required + +```typescript +public readonly openSearchEndpoint: string; +``` + +- *Type:* string + +The OpenSearch Cluster or Serverless collection endpoint to connect to. + +if you provisoned your cluster using CDK +use domainEndpoint property of OpenSearch provisioned cluster or +attrCollectionEndpoint property of OpenSearch Serverless collection. + +--- + +##### `iamHandlerRole`Optional + +```typescript +public readonly iamHandlerRole: IRole; +``` + +- *Type:* aws-cdk-lib.aws_iam.IRole +- *Default:* new IAMRole is created. + +The IAM role to pass to IAM authentication lambda handler This role must be able to be assumed with `lambda.amazonaws.com` service principal. + +--- + +##### `openSearchEndpointRegion`Optional + +```typescript +public readonly openSearchEndpointRegion: string; +``` + +- *Type:* string +- *Default:* same region as stack. + +AWS Region openSearchEndpoint is provisioned in. + +--- + +##### `removalPolicy`Optional + +```typescript +public readonly removalPolicy: RemovalPolicy; +``` + +- *Type:* aws-cdk-lib.RemovalPolicy +- *Default:* The resources are not deleted (`RemovalPolicy.RETAIN`). + +The removal policy when deleting the CDK resource. + +If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true. +Otherwise the removalPolicy is reverted to RETAIN. + +--- + +##### `subnets`Optional + +```typescript +public readonly subnets: SubnetSelection; +``` + +- *Type:* aws-cdk-lib.aws_ec2.SubnetSelection +- *Default:* One private subnet with egress is used per AZ. + +The subnets where the Custom Resource Lambda Function would be created in. + +Required if vpc parameter is provided. + +--- + +##### `vpc`Optional + +```typescript +public readonly vpc: IVpc; +``` + +- *Type:* aws-cdk-lib.aws_ec2.IVpc +- *Default:* no VPC is used. + +Defines the virtual networking environment for this construct. + +Typically should use same VPC as OpenSearch cluster or serverless collection. +Must have at least 2 subnets in two different AZs. + +--- + ### OpenSearchClusterProps Simplified configuration for the OpenSearch Cluster. @@ -19232,6 +19601,27 @@ Enum for MSK cluster types. --- +### OpenSearchClusterType + +#### Members + +| **Name** | **Description** | +| --- | --- | +| PROVISIONED | *No description.* | +| SERVERLESS | *No description.* | + +--- + +##### `PROVISIONED` + +--- + + +##### `SERVERLESS` + +--- + + ### OpenSearchNodes Default Node Instances for OpenSearch cluster. diff --git a/framework/src/consumption/examples/opensearch-api.lit.ts b/framework/src/consumption/examples/opensearch-api.lit.ts index 55e9afd80..be252fb64 100644 --- a/framework/src/consumption/examples/opensearch-api.lit.ts +++ b/framework/src/consumption/examples/opensearch-api.lit.ts @@ -8,23 +8,22 @@ class ExampleOpenSearchApiStack extends cdk.Stack { constructor(scope: Construct, id: string , props:cdk.StackProps) { super(scope, id, props); + this.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true); /// !show const osCluster = new dsf.consumption.OpenSearchCluster(this, 'MyOpenSearchCluster',{ domainName:"mycluster", samlEntityId:'', - samlMetadataContent:'', + samlMetadataContent:'', samlMasterBackendRole:'', deployInVpc:false, dataNodeInstanceType:'t3.small.search', dataNodeInstanceCount:1, - masterNodeInstanceCount:0 + masterNodeInstanceCount:0, + removalPolicy:cdk.RemovalPolicy.DESTROY }); - /// !hide //Add another admin - const adminCr = osCluster.addRoleMapping('AnotherAdmin', 'all_access','sometestId'); - //Overwrite construct-wide removal policy - adminCr.applyRemovalPolicy(cdk.RemovalPolicy.DESTROY); + osCluster.addRoleMapping('AnotherAdmin', 'all_access','sometestId'); const indexTemplateCr = osCluster.callOpenSearchApi('CreateIndexTemplate','_index_template/movies', { @@ -66,6 +65,7 @@ class ExampleOpenSearchApiStack extends cdk.Stack { add2Cr.node.addDependency(indexTemplateCr); const add3Cr = osCluster.callOpenSearchApi('AddData4', 'movies-01/_doc',{"title": "The Little Mermaid", "year": 2015}, 'POST'); add3Cr.node.addDependency(indexTemplateCr); + /// !hide } } diff --git a/framework/src/consumption/examples/opensearch-saml-clientvpn.lit.ts b/framework/src/consumption/examples/opensearch-saml-clientvpn.lit.ts index 26ea789c7..375460ec0 100644 --- a/framework/src/consumption/examples/opensearch-saml-clientvpn.lit.ts +++ b/framework/src/consumption/examples/opensearch-saml-clientvpn.lit.ts @@ -25,9 +25,11 @@ class ExampleDefaultOpenSearchStack extends cdk.Stack { deployInVpc:true, vpc:vpcVpn.vpc }); - /// !hide + osCluster.addRoleMapping('DashboardOsUser', 'dashboards_user',''); osCluster.addRoleMapping('ReadAllOsRole','readall',''); + /// !hide + } diff --git a/framework/src/consumption/examples/opensearch-saml.lit.ts b/framework/src/consumption/examples/opensearch-saml.lit.ts index ef06b17cf..902864466 100644 --- a/framework/src/consumption/examples/opensearch-saml.lit.ts +++ b/framework/src/consumption/examples/opensearch-saml.lit.ts @@ -17,9 +17,11 @@ class ExampleDefaultOpenSearchStack extends cdk.Stack { deployInVpc:true, removalPolicy:cdk.RemovalPolicy.DESTROY }); - /// !hide + + osCluster.addRoleMapping('DashboardOsUser', 'dashboards_user',''); osCluster.addRoleMapping('ReadAllOsRole','readall',''); + /// !hide } diff --git a/framework/src/consumption/lib/opensearch/index.ts b/framework/src/consumption/lib/opensearch/index.ts index 2bcac9d77..2ad9490b0 100644 --- a/framework/src/consumption/lib/opensearch/index.ts +++ b/framework/src/consumption/lib/opensearch/index.ts @@ -1,2 +1,4 @@ export * from './opensearch'; export * from './opensearch-props'; +export * from './opensearch-api-props'; +export * from './opensearch-api'; \ No newline at end of file diff --git a/framework/src/consumption/lib/opensearch/opensearch-api-props.ts b/framework/src/consumption/lib/opensearch/opensearch-api-props.ts new file mode 100644 index 000000000..8fa28ca81 --- /dev/null +++ b/framework/src/consumption/lib/opensearch/opensearch-api-props.ts @@ -0,0 +1,67 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { RemovalPolicy } from 'aws-cdk-lib'; +import { IVpc, SubnetSelection } from 'aws-cdk-lib/aws-ec2'; +import { IRole } from 'aws-cdk-lib/aws-iam'; + +/** + * Configuration for the OpenSearch API. + */ +export interface OpenSearchApiProps { + + /** + * The removal policy when deleting the CDK resource. + * If DESTROY is selected, context value `@data-solutions-framework-on-aws/removeDataOnDestroy` needs to be set to true. + * Otherwise the removalPolicy is reverted to RETAIN. + * @default - The resources are not deleted (`RemovalPolicy.RETAIN`). + */ + readonly removalPolicy?: RemovalPolicy; + + /** + * The IAM role to pass to IAM authentication lambda handler + * This role must be able to be assumed with `lambda.amazonaws.com` service principal + * @default - new IAMRole is created. + */ + readonly iamHandlerRole?: IRole; + + /** + * Defines the virtual networking environment for this construct. + * Typically should use same VPC as OpenSearch cluster or serverless collection. + * Must have at least 2 subnets in two different AZs. + * @default - no VPC is used. + */ + readonly vpc?: IVpc; + + /** + * The subnets where the Custom Resource Lambda Function would be created in. + * Required if vpc parameter is provided. + * @default - One private subnet with egress is used per AZ. + */ + readonly subnets?: SubnetSelection; + + /** + * The OpenSearch Cluster or Serverless collection endpoint to connect to. + * if you provisoned your cluster using CDK + * use domainEndpoint property of OpenSearch provisioned cluster or + * attrCollectionEndpoint property of OpenSearch Serverless collection. + */ + readonly openSearchEndpoint : string; + + /** + * Type of OpenSearch cluster. + */ + readonly openSearchClusterType : OpenSearchClusterType; + + /** + * AWS Region openSearchEndpoint is provisioned in. + * @default - same region as stack. + */ + readonly openSearchEndpointRegion? : string; +} + + +export enum OpenSearchClusterType { + PROVISIONED = 'provisioned', + SERVERLESS = 'serverless' +} \ No newline at end of file diff --git a/framework/src/consumption/lib/opensearch/opensearch-api.ts b/framework/src/consumption/lib/opensearch/opensearch-api.ts new file mode 100644 index 000000000..4f3f76e44 --- /dev/null +++ b/framework/src/consumption/lib/opensearch/opensearch-api.ts @@ -0,0 +1,126 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as path from 'path'; +import { CustomResource, Stack } from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { OpenSearchApiProps } from './opensearch-api-props'; +import { Context, TrackedConstruct, TrackedConstructProps } from '../../../utils'; +import { DsfProvider } from '../../../utils/lib/dsf-provider'; + +/** + * A construct to create an OpenSearch API client + */ +export class OpenSearchApi extends TrackedConstruct { + + /** + * Custom resource provider for the OpenSearch API client + */ + private apiProvider: DsfProvider; + + /** + * The removal policy when deleting the CDK resources. + * If DESTROY is selected, the context value '@data-solutions-framework-on-aws/removeDataOnDestroy' + * in the 'cdk.json' or 'cdk.context.json' must be set to true + * @default - The resources are not deleted (`RemovalPolicy.RETAIN`). + */ + private removalPolicy: any; + + /** + * Internal property to keep persistent IAM roles to prevent them from being overwritten + * in subsequent API calls + */ + private persistentRoles: {[key:string]:string[]} = {}; + + /** + * Constructs a new instance of the OpenSearch API construct. + * @param scope Construct + * @param id unique ID for the construct + * @param props @see OpenSearchApiProps + */ + constructor(scope: Construct, id: string, props: OpenSearchApiProps) { + + const trackedConstructProps: TrackedConstructProps = { + trackingTag: OpenSearchApi.name, + }; + + super(scope, id, trackedConstructProps); + + + this.removalPolicy = Context.revertRemovalPolicy(this, props.removalPolicy); + + this.apiProvider = new DsfProvider(this, 'Provider', { + providerName: 'opensearchApiProvider', + onEventHandlerDefinition: { + iamRole: props.iamHandlerRole, + handler: 'handler', + depsLockFilePath: path.join(__dirname, './resources/lambda/opensearch-api/package-lock.json'), + entryFile: path.join(__dirname, './resources/lambda/opensearch-api/opensearch-api.mjs'), + environment: { + REGION: props.openSearchEndpointRegion ?? Stack.of(this).region, + ENDPOINT: props.openSearchEndpoint, + }, + bundling: { + nodeModules: [ + '@aws-crypto/sha256-js', + '@aws-crypto/client-node', + '@aws-sdk/client-secrets-manager', + '@aws-sdk/node-http-handler', + '@aws-sdk/protocol-http', + '@aws-sdk/signature-v4', + ], + }, + }, + vpc: props.vpc, + subnets: props.subnets, + removalPolicy: this.removalPolicy, + }); + } + + + /** + * Calls OpenSearch API using custom resource. + * @param id The CDK resource ID + * @param apiPath OpenSearch API path + * @param body OpenSearch API request body + * @param method Opensearch API method, @default PUT + * @returns CustomResource object. + */ + + public callOpenSearchApi(id: string, apiPath: string, body: any, method?: string) : CustomResource { + const cr = new CustomResource(this, id, { + serviceToken: this.apiProvider.serviceToken, + resourceType: 'Custom::OpenSearchAPI', + properties: { + path: apiPath, + body, + method: method ?? 'PUT', + }, + removalPolicy: this.removalPolicy, + }); + cr.node.addDependency(this.apiProvider); + + return cr; + } + + /** + * @public + * Add a new role mapping to the cluster. + * This method is used to add a role mapping to the Amazon OpenSearch cluster + * @param id The CDK resource ID + * @param name OpenSearch role name @see https://opensearch.org/docs/2.9/security/access-control/users-roles/#predefined-roles + * @param role list of IAM roles. For IAM Identity center provide SAML group Id as a role + * @param persist Set to true if you want to prevent the roles to be ovewritten by subsequent PUT API calls. Default false. + * @returns CustomResource object. + */ + public addRoleMapping(id: string, name: string, role: string, persist:boolean=false) : CustomResource { + const persistentRoles = this.persistentRoles[name] || []; + const rolesToPersist = persistentRoles.concat([role]); + if (persist) { + this.persistentRoles[name] = rolesToPersist; + } + return this.callOpenSearchApi(id, '_plugins/_security/api/rolesmapping/' + name, { + backend_roles: rolesToPersist, + }); + } +} \ No newline at end of file diff --git a/framework/src/consumption/lib/opensearch/opensearch.ts b/framework/src/consumption/lib/opensearch/opensearch.ts index b730ebe71..c0086a5f1 100644 --- a/framework/src/consumption/lib/opensearch/opensearch.ts +++ b/framework/src/consumption/lib/opensearch/opensearch.ts @@ -1,8 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT-0 -import * as path from 'path'; -import { RemovalPolicy, CustomResource, Stack, Duration, Aws } from 'aws-cdk-lib'; +import { RemovalPolicy, Stack, Duration, Aws, CustomResource } from 'aws-cdk-lib'; import { EbsDeviceVolumeType, IVpc, Peer, Port, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { AnyPrincipal, Effect, IRole, ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { IKey, Key } from 'aws-cdk-lib/aws-kms'; @@ -10,9 +9,10 @@ import { ILogGroup, LogGroup } from 'aws-cdk-lib/aws-logs'; import { Domain, DomainProps, IDomain, SAMLOptionsProperty } from 'aws-cdk-lib/aws-opensearchservice'; import { AwsCustomResource, PhysicalResourceId } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; +import { OpenSearchApi } from './opensearch-api'; +import { OpenSearchClusterType } from './opensearch-api-props'; import { OpenSearchClusterProps, OpenSearchNodes, OPENSEARCH_DEFAULT_VERSION } from './opensearch-props'; import { Context, CreateServiceLinkedRole, DataVpc, TrackedConstruct, TrackedConstructProps } from '../../../utils'; -import { DsfProvider } from '../../../utils/lib/dsf-provider'; import { ServiceLinkedRoleService } from '../../../utils/lib/service-linked-role-service'; @@ -61,14 +61,15 @@ export class OpenSearchCluster extends TrackedConstruct { public readonly vpc:IVpc | undefined; /** - * CDK Custom resource provider for calling OpenSearch APIs + * IAM Role used to provision and configure OpenSearch domain */ - private readonly apiProvider: DsfProvider; + public readonly masterRole: IRole; /** - * IAM Role used to provision and configure OpenSearch domain + * Opesearch API client + * @see OpenSearchApi */ - public readonly masterRole: IRole; + private openSearchApi: OpenSearchApi; /** * The removal policy when deleting the CDK resource. @@ -79,15 +80,6 @@ export class OpenSearchCluster extends TrackedConstruct { */ private removalPolicy: RemovalPolicy; - /** - * Internal property to handle inital set of Acl API commands - */ - private aclAccessCr?: CustomResource[]; - - /** - * Internal property to keep persistent IAM roles to prevent them from being overwritten via API - */ - private persistentRoles: {[key:string]:string[]} = {}; /** * Constructs a new instance of the OpenSearchCluster class @@ -280,48 +272,29 @@ export class OpenSearchCluster extends TrackedConstruct { resources: [`arn:aws:es:${Aws.REGION}:${Aws.ACCOUNT_ID}:domain/${domain.domainName}/*`], })); - this.apiProvider = new DsfProvider(this, 'Provider', { - providerName: 'opensearchApiProvider', - onEventHandlerDefinition: { - iamRole: this.masterRole, - handler: 'handler', - depsLockFilePath: path.join(__dirname, './resources/lambda/opensearch-api/package-lock.json'), - entryFile: path.join(__dirname, './resources/lambda/opensearch-api/opensearch-api.mjs'), - environment: { - REGION: Stack.of(this).region, - ENDPOINT: domain.domainEndpoint, - }, - bundling: { - nodeModules: [ - '@aws-crypto/sha256-js', - '@aws-crypto/client-node', - '@aws-sdk/client-secrets-manager', - '@aws-sdk/node-http-handler', - '@aws-sdk/protocol-http', - '@aws-sdk/signature-v4', - ], - }, - }, - vpc: this.vpc, - subnets: subnets, - removalPolicy: this.removalPolicy, - }); this.domain = domain; const samlAdminGroupId = props.samlMasterBackendRole; - const allAccessRoleLambdaCr=this.addRoleMapping('AllAccessRoleLambda', 'all_access', this.masterRole.roleArn, true); - const allAccessRoleSamlCr=this.addRoleMapping('AllAccessRoleSAML', 'all_access', samlAdminGroupId, true); + this.openSearchApi = new OpenSearchApi(scope, 'OpenSearchApi', { + openSearchEndpoint: this.domain.domainEndpoint, + openSearchClusterType: OpenSearchClusterType.PROVISIONED, + iamHandlerRole: this.masterRole, + vpc: this.vpc, + subnets: subnets, + removalPolicy: this.removalPolicy, + }); + + const allAccessRoleLambdaCr=this.openSearchApi.addRoleMapping('AllAccessRoleLambda', 'all_access', this.masterRole.roleArn, true); + const allAccessRoleSamlCr=this.openSearchApi.addRoleMapping('AllAccessRoleSAML', 'all_access', samlAdminGroupId, true); allAccessRoleSamlCr.node.addDependency(allAccessRoleLambdaCr); - const securityManagerRoleLambdaCr = this.addRoleMapping('SecurityManagerRoleLambda', 'security_manager', this.masterRole.roleArn, true); + const securityManagerRoleLambdaCr = this.openSearchApi.addRoleMapping('SecurityManagerRoleLambda', 'security_manager', this.masterRole.roleArn, true); securityManagerRoleLambdaCr.node.addDependency(allAccessRoleSamlCr); - const securityManagerRoleSamlCr = this.addRoleMapping('SecurityManagerRoleSAML', 'security_manager', samlAdminGroupId, true); + const securityManagerRoleSamlCr = this.openSearchApi.addRoleMapping('SecurityManagerRoleSAML', 'security_manager', samlAdminGroupId, true); securityManagerRoleSamlCr.node.addDependency(securityManagerRoleLambdaCr); - this.aclAccessCr=[allAccessRoleLambdaCr, allAccessRoleSamlCr, securityManagerRoleLambdaCr, securityManagerRoleSamlCr]; - //enable SAML authentication after adding lambda permissions to execute API calls later. const updateDomain = new AwsCustomResource(this, 'EnableInternalUserDatabaseCR', { installLatestAwsSdk: false, @@ -352,8 +325,10 @@ export class OpenSearchCluster extends TrackedConstruct { removalPolicy: this.removalPolicy, }); updateDomain.node.addDependency(this.domain); - for (const aclCr of this.aclAccessCr) updateDomain.node.addDependency(aclCr); - + for (const aclCr of + [allAccessRoleLambdaCr, allAccessRoleSamlCr, securityManagerRoleLambdaCr, securityManagerRoleSamlCr]) { + updateDomain.node.addDependency(aclCr); + } } /** @@ -366,27 +341,7 @@ export class OpenSearchCluster extends TrackedConstruct { */ public callOpenSearchApi(id: string, apiPath: string, body: any, method?: string) : CustomResource { - const cr = new CustomResource(this, id, { - serviceToken: this.apiProvider.serviceToken, - resourceType: 'Custom::OpenSearchAPI', - properties: { - path: apiPath, - body, - method: method ?? 'PUT', - }, - removalPolicy: this.removalPolicy, - }); - cr.node.addDependency(this.domain); - cr.node.addDependency(this.apiProvider); - - //add aclAccessCr dependencies for user-defined Api calls. - if (this.aclAccessCr?.length && this.aclAccessCr.map((aclCr)=> aclCr.node.id).indexOf(cr.node.id)<0) { - for (const aclCr of this.aclAccessCr) { - cr.node.addDependency(aclCr); - } - } - - return cr; + return this.openSearchApi.callOpenSearchApi(id, apiPath, body, method); } /** @@ -399,15 +354,7 @@ export class OpenSearchCluster extends TrackedConstruct { * @param persist Set to true if you want to prevent the roles to be ovewritten by subsequent PUT API calls. Default false. * @returns CustomResource object. */ - public addRoleMapping(id: string, name: string, role: string, persist:boolean=false) : CustomResource { - const persistentRoles = this.persistentRoles[name] || []; - const rolesToPersist = persistentRoles.concat([role]); - if (persist) { - this.persistentRoles[name] = rolesToPersist; - } - return this.callOpenSearchApi(id, '_plugins/_security/api/rolesmapping/' + name, { - backend_roles: rolesToPersist, - }); + return this.openSearchApi.addRoleMapping(id, name, role, persist); } } \ No newline at end of file diff --git a/framework/test/e2e/opensearch-api.e2e.test.ts b/framework/test/e2e/opensearch-api.e2e.test.ts new file mode 100644 index 000000000..1549f465e --- /dev/null +++ b/framework/test/e2e/opensearch-api.e2e.test.ts @@ -0,0 +1,71 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * E2E test for OpenSearch + * + * @group e2e/consumption/opensearch-api + */ + + +import { App, RemovalPolicy, CfnOutput } from 'aws-cdk-lib'; + +import { TestStack } from './test-stack'; +import { OpenSearchCluster } from '../../src/consumption/index'; + + +jest.setTimeout(10000000); + +// GIVEN +const app = new App(); +const testStack = new TestStack('OpenSearchApiTestStack', app); +const { stack } = testStack; +stack.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true); + +const domain = new OpenSearchCluster(stack, 'OpenSearchVpc', { + domainName: 'e2e-tests-cluster', + samlEntityId: 'https://portal.sso.eu-west-1.amazonaws.com/saml/assertion/MTQ1Mzg4NjI1ODYwX2lucy02MmQ3Y2VlYWM0YWNkNjA1', + samlMetadataContent: ` + + + + + MIIDBzCCAe+gAwIBAgIFAMWCViwwDQYJKoZIhvcNAQELBQAwRTEWMBQGA1UEAwwNYW1hem9uYXdzLmNvbTENMAsGA1UECwwESURBUzEPMA0GA1UECgwGQW1hem9uMQswCQYDVQQGEwJVUzAeFw0yNDAyMjExNTQ4MTJaFw0yOTAyMjExNTQ4MTJaMEUxFjAUBgNVBAMMDWFtYXpvbmF3cy5jb20xDTALBgNVBAsMBElEQVMxDzANBgNVBAoMBkFtYXpvbjELMAkGA1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDHco1Vg9SeXOJHv4CEbY7folO9+zP3naA570q97Oi9o81l9ibP5c+a1404qUBuv5E4HH1chtHU6Yos5LMXHaRet/bOUBrAIOieF0RCMLfHz1Vkf213SWf60yfAA19QgwQRLH3HTEc+nhfe93RAgcw1T7mZBcHk5Zljt6gShq3N4YzupO4KpuRBX2S2XzfhasuDV8JpcB6BGexTDzAEcZ1P0v8X+vpCF7fN9Gd5K/rrOtCuPcSC694KJgfOucvMNj7PqpkLvLzTosxlqL6P5PQheW5sZwYqvw+MJrGIg5WBqRXoTF0JE5A6lv1aWhQDfuyzQ8UojGeMjTwgz1/PTQGFAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADWTz+ggtrkhDGYKqEqFn04s0fMhfcQ9f6j0Rs8igysdINM7VCyD4PJapn5kekKwlzir27t9fpCD+PcgiCAKxGnaTvaKSTfoyGvHnRYhTbjb+XougPyyTl5qdJZkXx0x6ucw6OjbF/WH2VLY1xvr9MQkbUWNcS9b2FDIujHs881hpITPIKadV42BbIAK5sRJTncykJ6KSdN/MGVwYYVrE2rAM1uubwcKLkmbGxDBiS7ci3gu0M+5A53WHrjeGR/JC6ER49ybGYjouEKJWRw8ixRLV03H1kdrveuV4CUdv/mUJLzHulHWEGZFrvJIrQjf0ORYb790AAjBg092tlwg2Ys= + + + + + + + + + +`, + samlMasterBackendRole: 'admin', + deployInVpc: false, + removalPolicy: RemovalPolicy.DESTROY, +}); + + +const cr = domain.addRoleMapping('DashboardOsUser', 'dashboards_user', ''); + +new CfnOutput(stack, 'OpenSearchApiCr', { + value: cr.toString(), +}); + + +let deployResult: Record; + +beforeAll(async() => { + // WHEN + deployResult = await testStack.deploy(); +}, 10000000); + +it('Custom resource created successfully', async () => { + // THEN + expect(deployResult.OpenSearchApiCr).toContain('OpenSearchApiTestStack'); +}); + +afterAll(async () => { + await testStack.destroy(); +}, 10000000); diff --git a/framework/test/unit/consumption/opensearch-api.test.ts b/framework/test/unit/consumption/opensearch-api.test.ts new file mode 100644 index 000000000..f76431d63 --- /dev/null +++ b/framework/test/unit/consumption/opensearch-api.test.ts @@ -0,0 +1,53 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 + +/** + * Tests for OpenSearch cluster API construct + * @group unit/consumption/opensearch-api + */ + + +import { Stack, App, RemovalPolicy } from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { OpenSearchCluster } from '../../../src/consumption'; + + +describe('default configuration', () => { + + const app = new App(); + const stack = new Stack(app, 'Stack'); + + // Set context value for global data removal policy + stack.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true); + + // Instantiate AccessLogsBucket Construct with default + const osCluster = new OpenSearchCluster(stack, 'OpenSearchTestApi', { + domainName: 'mycluster2', + samlEntityId: '', + samlMetadataContent: 'xmlContent', + samlMasterBackendRole: 'IdpGroupId', + deployInVpc: false, + removalPolicy: RemovalPolicy.DESTROY, + }); + + osCluster.addRoleMapping('role1', 'user1', 'group1'); + const template = Template.fromStack(stack); + + test('should have OpenSearch domain', () => { + template.resourceCountIs('AWS::OpenSearchService::Domain', 1); + }); + + test('creates opensearch api client', () => { + template.hasResourceProperties('AWS::Lambda::Function', { + Timeout: 840, + Tags: Match.arrayWith([ + { + Key: 'data-solutions-fwk:owned', + Value: 'true', + }, + ]), + }); + }); + +}); + diff --git a/framework/test/unit/nag/consumption/nag-opensearch-api.test.ts b/framework/test/unit/nag/consumption/nag-opensearch-api.test.ts new file mode 100644 index 000000000..27dc1936c --- /dev/null +++ b/framework/test/unit/nag/consumption/nag-opensearch-api.test.ts @@ -0,0 +1,155 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 + + +/** + * Tests OpenSearch API cluster construct + * + * @group unit/nag/consumption/opensearch-api + */ + +import { App, Aspects, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { Annotations, Match } from 'aws-cdk-lib/assertions'; +import { AwsSolutionsChecks, NagSuppressions } from 'cdk-nag'; +import { OpenSearchCluster } from '../../../../src/consumption'; + + +const app = new App(); +const stack = new Stack(app, 'Stack'); + +stack.node.setContext('@data-solutions-framework-on-aws/removeDataOnDestroy', true); + +// Instantiate AccessLogsBucket Construct with default +const osCluster = new OpenSearchCluster(stack, 'OpenSearch', { + domainName: 'test', + masterNodeInstanceCount: 3, + dataNodeInstanceCount: 4, + samlEntityId: '', + samlMetadataContent: 'xmlCOntent', + samlMasterBackendRole: 'IdpGroupId', + deployInVpc: true, + removalPolicy: RemovalPolicy.DESTROY, +}); + +osCluster.addRoleMapping('testMapping', 'test', 'test'); + +Aspects.of(stack).add(new AwsSolutionsChecks({ verbose: true })); + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/OpenSearch/SecurityGroup/Resource', + [{ id: 'CdkNagValidationFailure', reason: 'VPC can be created or supplied as props, so cidr block is not known in advance' }], +); +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/OpenSearch/AccessRole/Resource', + [{ id: 'AwsSolutions-IAM4', reason: 'this is default recommended IAM Role to use' }], +); + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/CreateSLR/Provider/CustomResourceProvider/framework-onEvent/ServiceRole/Resource', + [{ id: 'AwsSolutions-IAM4', reason: 'Separately handled in SLR construct' }], +); + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/CreateSLR/Provider/CustomResourceProvider/framework-onEvent/ServiceRole/DefaultPolicy/Resource', + [{ id: 'AwsSolutions-IAM5', reason: 'Separately handled in SLR construct' }], +); + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/CreateSLR/Provider/CustomResourceProvider/framework-onEvent/Resource', + [{ id: 'AwsSolutions-L1', reason: 'Separately handled in SLR construct' }], +); + + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/AWS679f53fac002430cb0da5b7982bd2287/ServiceRole/Resource', + [ + { id: 'AwsSolutions-IAM4', reason: 'AWSLambdaBasicExecutionRole this is default recommended IAM Policy to use' }, + ], +); +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a/ServiceRole/Resource', + [ + { id: 'AwsSolutions-IAM4', reason: 'AWSLambdaBasicExecutionRole this is default recommended IAM Policy to use' }, + ], +); +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a/ServiceRole/DefaultPolicy/Resource', + [ + { id: 'AwsSolutions-IAM5', reason: 'The policy is provided by the Custom Resource framework and can\'t be updated' }, + ], +); + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/AWS679f53fac002430cb0da5b7982bd2287/Resource', + [ + { id: 'AwsSolutions-L1', reason: 'Part of the Custom Resource framework and can\'t be updated' }, + ], +); + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/OpenSearch/Domain/ESLogGroupPolicyc8fcef8a0773977d390b894952b755dcb5cb887768/CustomResourcePolicy/Resource', + [{ id: 'AwsSolutions-IAM5', reason: 'this is default recommended IAM Role to use' }], +); + + +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/OpenSearch/MasterRolePolicy/Resource', + [ + { id: 'AwsSolutions-IAM5', reason: 'Role needs access to all OpenSearch APIs' }, + { id: 'AwsSolutions-IAM4', reason: 'Role needs access to all OpenSearch APIs' }, + ], +); + +//recommendaed FGAC https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html +NagSuppressions.addResourceSuppressionsByPath( + stack, + '/Stack/OpenSearch/Domain/Resource', + [ + { id: 'AwsSolutions-OS3', reason: 'SAML authentication is used to restrict access' }, + { id: 'AwsSolutions-OS5', reason: 'IAM-based access and SAML authentication for dashboards are used to restrict access' }, + ], +); + + +NagSuppressions.addResourceSuppressionsByPath( + stack, [ + '/Stack/OpenSearchApi/Provider/VpcPolicy/Resource', + '/Stack/OpenSearchApi/Provider/CleanUpProvider', + '/Stack/OpenSearchApi/Provider/CustomResourceProvider/framework-onEvent', + ], + [ + { id: 'AwsSolutions-IAM5', reason: 'Resource is not part of the test scope' }, + { id: 'AwsSolutions-IAM4', reason: 'Resource is not part of the test scope' }, + { id: 'AwsSolutions-L1', reason: 'Resource is not part of the test scope' }, + ], + true, +); + + +test('No unsuppressed Warnings', () => { + const warnings = Annotations.fromStack(stack).findWarning('*', Match.stringLikeRegexp('AwsSolutions-.*')); + console.log(warnings); + expect(warnings).toHaveLength(0); +}); + +test('No unsuppressed Errors', () => { + const errors = Annotations.fromStack(stack).findError('*', Match.stringLikeRegexp('AwsSolutions-.*')); + for (const error of errors) { + expect(error.id+' '+error.entry.data).toHaveLength(0); + console.log(error.id); + } + console.log(errors); + expect(errors).toHaveLength(0); +}); + diff --git a/framework/test/unit/nag/consumption/nag-opensearch.test.ts b/framework/test/unit/nag/consumption/nag-opensearch.test.ts index 59fad3e53..d95e75f6c 100644 --- a/framework/test/unit/nag/consumption/nag-opensearch.test.ts +++ b/framework/test/unit/nag/consumption/nag-opensearch.test.ts @@ -122,9 +122,9 @@ NagSuppressions.addResourceSuppressionsByPath( NagSuppressions.addResourceSuppressionsByPath( stack, [ - '/Stack/OpenSearch/Provider/VpcPolicy/Resource', - '/Stack/OpenSearch/Provider/CleanUpProvider', - '/Stack/OpenSearch/Provider/CustomResourceProvider/framework-onEvent', + '/Stack/OpenSearchApi/Provider/VpcPolicy/Resource', + '/Stack/OpenSearchApi/Provider/CleanUpProvider', + '/Stack/OpenSearchApi/Provider/CustomResourceProvider/framework-onEvent', ], [ { id: 'AwsSolutions-IAM5', reason: 'Resource is not part of the test scope' }, diff --git a/website/docs/constructs/library/generated/_consumption-opensearch.mdx b/website/docs/constructs/library/generated/_consumption-opensearch.mdx index 62ea6f9d5..8351adec9 100644 --- a/website/docs/constructs/library/generated/_consumption-opensearch.mdx +++ b/website/docs/constructs/library/generated/_consumption-opensearch.mdx @@ -49,14 +49,18 @@ Default configuration ```typescript -const osCluster = new dsf.consumption.OpenSearchCluster(this, 'MyOpenSearchCluster',{ - domainName:"mycluster", - samlEntityId:'', - samlMetadataContent:'', - samlMasterBackendRole:'', - deployInVpc:true, - removalPolicy:cdk.RemovalPolicy.DESTROY -}); + const osCluster = new dsf.consumption.OpenSearchCluster(this, 'MyOpenSearchCluster',{ + domainName:"mycluster", + samlEntityId:'', + samlMetadataContent:'', + samlMasterBackendRole:'', + deployInVpc:true, + removalPolicy:cdk.RemovalPolicy.DESTROY + }); + + + osCluster.addRoleMapping('DashboardOsUser', 'dashboards_user',''); + osCluster.addRoleMapping('ReadAllOsRole','readall',''); ``` @@ -71,6 +75,9 @@ os_cluster = dsf.consumption.OpenSearchCluster(self, "MyOpenSearchCluster", deploy_in_vpc=True, removal_policy=cdk.RemovalPolicy.DESTROY ) + +os_cluster.add_role_mapping("DashboardOsUser", "dashboards_user", "") +os_cluster.add_role_mapping("ReadAllOsRole", "readall", "") ``` @@ -82,22 +89,25 @@ Using Client VPN Endpoint ```typescript -const vpcVpn = new dsf.utils.DataVpc(this, 'VpcWithVpn', { - vpcCidr:'10.0.0.0/16', - clientVpnEndpointProps: { - serverCertificateArn:"", - samlMetadataDocument:``, - selfServicePortal:false - } -}) -const osCluster = new dsf.consumption.OpenSearchCluster(this, 'MyOpenSearchCluster',{ - domainName:"mycluster", - samlEntityId:'', - samlMetadataContent:'', - samlMasterBackendRole:'', - deployInVpc:true, - vpc:vpcVpn.vpc -}); + const vpcVpn = new dsf.utils.DataVpc(this, 'VpcWithVpn', { + vpcCidr:'10.0.0.0/16', + clientVpnEndpointProps: { + serverCertificateArn:"", + samlMetadataDocument:``, + selfServicePortal:false + } + }) + const osCluster = new dsf.consumption.OpenSearchCluster(this, 'MyOpenSearchCluster',{ + domainName:"mycluster", + samlEntityId:'', + samlMetadataContent:'', + samlMasterBackendRole:'', + deployInVpc:true, + vpc:vpcVpn.vpc + }); + + osCluster.addRoleMapping('DashboardOsUser', 'dashboards_user',''); + osCluster.addRoleMapping('ReadAllOsRole','readall',''); ``` @@ -120,6 +130,9 @@ os_cluster = dsf.consumption.OpenSearchCluster(self, "MyOpenSearchCluster", deploy_in_vpc=True, vpc=vpc_vpn.vpc ) + +os_cluster.add_role_mapping("DashboardOsUser", "dashboards_user", "") +os_cluster.add_role_mapping("ReadAllOsRole", "readall", "") ```