Async SDK client init (#632)

* Async AWS SDK client init. SDK client connection init now happens in a background task while runtime is loading and resolving modules * Don't wait for latch when 0 * Update SDK * Update test runner * Cleanup get_basename_ext_name * Also allow for lib- imports * Prevent duplicate client inits
awslabs · Oct 18, 2024 · fe7e483 · fe7e483
1 parent 63a952c
commit fe7e483
Show file tree

Hide file tree

Showing 19 changed files with 3,726 additions and 4,012 deletions.
diff --git a/Makefile b/Makefile
@@ -180,6 +180,8 @@ test: export JS_MINIFY = 0
 test: js
 	cargo run -- test -d bundle/js/__tests__/unit
 test-e2e: export JS_MINIFY = 0
+test-e2e: export TEST_TIMEOUT = 60000
+test-e2e: export SDK_BUNDLE_MODE = STD
 test-e2e: js
 	cargo run -- test -d bundle/js/__tests__/e2e
 

diff --git a/build.mjs b/build.mjs
@@ -40,12 +40,7 @@ const AWS_JSON_SHARED_COMMAND_REGEX2 =
   /{\s*const\s*headers\s*=\s*sharedHeaders\(("\w+")\);\s*let body;\s*body\s*=\s*JSON.stringify\((\w+)\(input,\s*context\)\);\s*return buildHttpRpcRequest\(context,\s*headers,\s*"\/",\s*undefined,\s*body\);\s*}/gm;
 const MINIFY_JS = process.env.JS_MINIFY !== "0";
 const SDK_UTILS_PACKAGE = "sdk-utils";
-const ENTRYPOINTS = [
-  "@llrt/std",
-  "stream",
-  "@llrt/test/index",
-  "@llrt/test/worker",
-];
+const ENTRYPOINTS = ["stream", "@llrt/test/index", "@llrt/test/worker"];
 
 const ES_BUILD_OPTIONS = {
   splitting: MINIFY_JS,
@@ -81,162 +76,7 @@ const ES_BUILD_OPTIONS = {
   ],
 };
 
-// API Endpoint Definitions
-//
-// const SDK_DATA = {
-//   // Classification
-//   "PackageName": ["ClientName", "ServiceEndpoint", fullSdkOnly],
-// }
-//
-// The meanings of each and how to look them up are as follows.
-//
-//   1. Classification
-//   https://docs.aws.amazon.com/whitepapers/latest/aws-overview/amazon-web-services-cloud-platform.html
-//
-//   2. ClientName
-//   https://www.npmjs.com/search?q=%40aws-sdk%2Fclient-
-//
-//    Case) @aws-sdk/client-sts
-//    [Import Section](https://www.npmjs.com/package/@aws-sdk/client-sts#getting-started)
-//
-//    > // ES6+ example
-//    > import { STSClient, GetCallerIdentityCommand } from "@aws-sdk/client-sts";
-//               ^^^ <- This part except for the "client"
-//
-//   3. ServiceEndpoint
-//   https://docs.aws.amazon.com/general/latest/gr/aws-service-information.html
-//
-//    Case) @aws-sdk/client-sts
-//    [Service endpoints Section](https://docs.aws.amazon.com/general/latest/gr/sts.html#sts_region)
-//
-//    > | Region Name    | Region    | Endpoint                    | Protocol |
-//    > | -------------- | --------- | --------------------------- | -------- |
-//    > | US East (Ohio) | us-east-2 | sts.us-east-2.amazonaws.com | HTTPS    |
-//                                     ^^^ <- String before "region"
-//
-//    If multiple endpoints are required, such as `states` and `sync-states` in @aws-sdk/client-sfn,
-//    multiple endpoints can be specified by making them an array.
-//
-//   4. fullSdkOnly
-//
-//     If you want to bundle only 'full-sdk', specify `true`.
-//     Specify `false` if you want to bundle for both 'std-sdk' and 'full-sdk'.
-//
-//     The combination of SDK_BUNDLE_MODE and fullSdkOnly automatically determines whether the bundle is eligible or not.
-//     Note that if SDK_BUNDLE_MODE is 'NONE', the above values are completely ignored and any SDKs are excluded from the bundle.
-//
-const SDK_DATA = {
-  // Analytics
-  "client-athena": ["Athena", "athena", true],
-  "client-firehose": ["Firehose", "firehose", true],
-  "client-glue": ["Glue", "glue", true],
-  "client-kinesis": ["Kinesis", "kinesis", true],
-  "client-opensearch": ["OpenSearch", "es", true],
-  "client-opensearchserverless": ["OpenSearchServerless", "aoss", true],
-  // ApplicationIntegration
-  "client-eventbridge": ["EventBridge", "events", false],
-  "client-scheduler": ["Scheduler", "scheduler", true],
-  "client-sfn": ["SFN", ["states", "sync-states"], false],
-  "client-sns": ["SNS", "sns", false],
-  "client-sqs": ["SQS", "sqs", false],
-  // Blockchain
-  ...{},
-  // BusinessApplications
-  "client-ses": ["SES", "email", false],
-  "client-sesv2": ["SESv2", "email", true],
-  // CloudFinancialManagement
-  ...{},
-  // ComputeServices
-  "client-auto-scaling": ["AutoScaling", "autoscaling", true],
-  "client-batch": ["Batch", "batch", true],
-  "client-ec2": ["EC2", "ec2", true],
-  "client-lambda": ["Lambda", "lambda", false],
-  // CustomerEnablement
-  ...{},
-  // Containers
-  "client-ecr": ["ECR", "api.ecr", true],
-  "client-ecs": ["ECS", "ecs", true],
-  "client-eks": ["EKS", "eks", true],
-  "client-servicediscovery": ["ServiceDiscovery", "discovery", true],
-  // Databases
-  "client-dynamodb": ["DynamoDB", "dynamodb", false],
-  "client-dynamodb-streams": ["DynamoDBStreams", "streams.dynamodb", true],
-  "client-elasticache": ["ElastiCache", "elasticache", true],
-  "client-rds": ["RDS", "rds", true],
-  "client-rds-data": ["RDSData", "rds-data", true],
-  "lib-dynamodb": ["DynamoDBDocument", "dynamodb", false],
-  // DeveloperTools
-  "client-xray": ["XRay", "xray", false],
-  // EndUserComputing
-  ...{},
-  // FrontendWebAndMobileServices
-  "client-amplify": ["Amplify", "amplify", true],
-  "client-appsync": ["AppSync", "appsync", true],
-  "client-location": ["Location", "geo", true],
-  // GameTech
-  ...{},
-  // InternetOfThings
-  ...{},
-  // MachineLearningAndArtificialIntelligence
-  "client-bedrock": ["Bedrock", "bedrock", true],
-  "client-bedrock-agent": ["BedrockAgent", "bedrock-agent", true],
-  "client-bedrock-runtime": ["BedrockRuntime", "bedrock-runtime", true],
-  "client-bedrock-agent-runtime": [
-    "BedrockAgentRuntime",
-    "bedrock-agent-runtime",
-    true,
-  ],
-  "client-polly": ["Polly", "polly", true],
-  "client-rekognition": ["Rekognition", "rekognition", true],
-  "client-textract": ["Textract", "textract", true],
-  "client-translate": ["Translate", "translate", true],
-  // ManagementAndGovernance
-  "client-appconfig": ["AppConfig", "appconfig", true],
-  "client-appconfigdata": ["AppConfigData", "appconfigdata", true],
-  "client-cloudformation": ["CloudFormation", "cloudformation", true],
-  "client-cloudwatch": ["CloudWatch", "monitoring", true],
-  "client-cloudwatch-logs": ["CloudWatchLogs", "logs", false],
-  "client-cloudwatch-events": ["CloudWatchEvents", "events", false],
-  "client-service-catalog": ["ServiceCatalog", "servicecatalog", true],
-  "client-ssm": ["SSM", "ssm", false],
-  // Media
-  "client-mediaconvert": ["MediaConvert", "mediaconvert", true],
-  // MigrationAndTransfer
-  ...{},
-  // NetworkingAndContentDelivery
-  "client-api-gateway": ["APIGateway", "apigateway", true],
-  "client-apigatewayv2": ["ApiGatewayV2", "apigateway", true],
-  "client-elastic-load-balancing-v2": [
-    "ElasticLoadBalancingV2",
-    "elasticloadbalancing",
-    true,
-  ],
-  // QuantumTechnologies
-  ...{},
-  // Robotics
-  ...{},
-  // Satellite
-  ...{},
-  // SecurityIdentityAndCompliance
-  "client-acm": ["ACM", "acm", true],
-  "client-cognito-identity": ["CognitoIdentity", "cognito-identity", false],
-  "client-cognito-identity-provider": [
-    "CognitoIdentityProvider",
-    "cognito-idp",
-    false,
-  ],
-  "client-iam": ["IAM", "iam", true],
-  "client-kms": ["KMS", "kms", false],
-  "client-secrets-manager": ["SecretsManager", "secretsmanager", false],
-  "client-sso": ["SSO", "identitystore", true],
-  "client-sso-admin": ["SSOAdmin", "identitystore", true],
-  "client-sso-oidc": ["SSOOIDC", "identitystore", true],
-  "client-sts": ["STS", "sts", false],
-  // Storage
-  "client-efs": ["EFS", "elasticfilesystem", true],
-  "client-s3": ["S3", "s3", false],
-  "lib-storage": ["Upload", "s3", false],
-};
+const SDK_DATA = await parseSdkData();
 
 const ADDITIONAL_PACKAGES = [
   "@aws-sdk/core",
@@ -307,6 +147,36 @@ Object.keys(SDK_DATA).forEach((sdk) => {
   }
 });
 
+async function parseSdkData() {
+  const cfgData = await fs.readFile("sdk.cfg");
+  const cfgLines = cfgData.toString().split("\n");
+
+  const sdkData = {};
+
+  for (let line of cfgLines) {
+    line = line.trim();
+    if (line.startsWith("#") || line == "") {
+      continue;
+    }
+
+    // Parse the line
+    const parts = line.split(",");
+
+    //get and remove the item at 0
+    const packageName = parts.shift();
+    const clientName = parts.shift();
+
+    //get and remove the last item
+    const fullSdkOnly = parts.pop() == 1;
+
+    const endpoints = parts;
+
+    // Log or store parsed information
+    sdkData[packageName] = [clientName, endpoints, fullSdkOnly];
+  }
+  return sdkData;
+}
+
 function resolveDefaultsModeConfigWrapper(config) {
   if (!config.credentials) {
     config.credentials = {
@@ -706,27 +576,10 @@ async function buildSdks() {
       const packagePath = path.join(TMP_DIR, pkg);
       const sdk = SDKS_BY_SDK_PACKAGES[pkg];
       const sdkIndexFile = path.join(packagePath, "index.js");
-      const value = SERVICE_ENDPOINTS_BY_PACKAGE[sdk];
-      const serviceNames = Array.isArray(value)
-        ? value
-        : value
-          ? [value]
-          : null;
+
       await fs.mkdir(packagePath, { recursive: true });
 
       let sdkContents = `export * from "${pkg}";`;
-      if (serviceNames) {
-        sdkContents += "\nif(__bootstrap.addAwsSdkInitTask){\n";
-        for (const serviceName of serviceNames) {
-          if (!serviceName) {
-            throw new Error(
-              `Service name "${serviceName}" for ${pkg} is invalid`
-            );
-          }
-          sdkContents += `__bootstrap.addAwsSdkInitTask("${serviceName}");\n`;
-        }
-        sdkContents += "}\n";
-      }
       await fs.writeFile(sdkIndexFile, sdkContents);
 
       return [pkg, sdkIndexFile];

diff --git a/example/functions/package.json b/example/functions/package.json
@@ -7,17 +7,17 @@
     "build": "node build.mjs"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "3.632.0",
-    "@aws-sdk/client-ec2": "3.632.0",
-    "@aws-sdk/client-s3": "3.633.0",
-    "aws-sdk": "2.1687.0",
+    "@aws-sdk/client-dynamodb": "3.670.0",
+    "@aws-sdk/client-ec2": "3.670.0",
+    "@aws-sdk/client-s3": "3.670.0",
+    "aws-sdk": "2.1691.0",
     "esbuild-css-modules-plugin": "3.1.2",
     "react": "18.3.1",
     "react-dom": "18.3.1"
   },
   "devDependencies": {
-    "@types/react": "18.3.10",
-    "@types/react-dom": "18.3.0",
-    "esbuild": "0.23.1"
+    "@types/react": "18.3.11",
+    "@types/react-dom": "18.3.1",
+    "esbuild": "0.24.0"
   }
 }
diff --git a/example/infrastructure/package.json b/example/infrastructure/package.json
@@ -9,12 +9,12 @@
     "hotswap": "cdk deploy --hotswap"
   },
   "devDependencies": {
-    "@types/node": "22.7.4",
-    "aws-cdk": "2.160.0",
-    "aws-cdk-lib": "2.160.0",
-    "constructs": "10.3.0",
-    "esbuild": "0.23.1",
+    "@types/node": "22.7.6",
+    "aws-cdk": "2.162.1",
+    "aws-cdk-lib": "2.162.1",
+    "constructs": "10.4.2",
+    "esbuild": "0.24.0",
     "ts-node": "10.9.2",
-    "typescript": "5.5.4"
+    "typescript": "5.6.3"
   }
 }
diff --git a/llrt_core/Cargo.toml b/llrt_core/Cargo.toml
@@ -94,7 +94,6 @@ rquickjs = { git = "https://github.com/DelSkayn/rquickjs.git", rev = "3af3f46b13
     "full-async",
     "rust-alloc",
 ], default-features = false }
-tokio = { version = "1", features = ["full"] }
 phf_codegen = "0.11.2"
 jwalk = "0.8.1"
 nanoid = "0.4.0"