From e2df3811df2e4a2363c81ef47ff8e0983ba3faf6 Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:40:08 +0100 Subject: [PATCH 01/21] container cannot write to the root filesystem --- .../service-catalogue.test.ts.snap | 24 +++++++++++++++++++ packages/cdk/lib/cloudquery/task.ts | 1 + 2 files changed, 25 insertions(+) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index ef55fefa0..3d499555d 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -191,6 +191,7 @@ spec: }, }, "Name": "CloudquerySource-AwsCostExplorerContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -855,6 +856,7 @@ spec: }, }, "Name": "CloudquerySource-DelegatedToSecurityAccountContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -1472,6 +1474,7 @@ spec: }, }, "Name": "CloudquerySource-DeployToolsListOrgsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -2083,6 +2086,7 @@ spec: }, }, "Name": "CloudquerySource-FastlyServicesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "FASTLY_API_KEY", @@ -2717,6 +2721,7 @@ spec: }, }, "Name": "CloudquerySource-GalaxiesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -3573,6 +3578,7 @@ spec: }, }, "Name": "CloudquerySource-GitHubIssuesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_PRIVATE_KEY", @@ -4015,6 +4021,7 @@ spec: }, }, "Name": "CloudquerySource-GitHubLanguagesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_ACCESS_TOKEN", @@ -4620,6 +4627,7 @@ spec: }, }, "Name": "CloudquerySource-GitHubRepositoriesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_PRIVATE_KEY", @@ -5291,6 +5299,7 @@ spec: }, }, "Name": "CloudquerySource-GitHubTeamsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_PRIVATE_KEY", @@ -5953,6 +5962,7 @@ spec: }, }, "Name": "CloudquerySource-NS1Container", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "NS1_API_KEY", @@ -6613,6 +6623,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideAutoScalingGroupsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -7233,6 +7244,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideBackupContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -7851,6 +7863,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideCertificatesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -8499,6 +8512,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideCloudFormationContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -9321,6 +9335,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideCloudwatchAlarmsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -9705,6 +9720,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideDynamoDBContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -10329,6 +10345,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideEc2Container", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -10974,6 +10991,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideInspectorContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -11827,6 +11845,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideLoadBalancersContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -12448,6 +12467,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideRDSContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -12832,6 +12852,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideS3Container", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -13513,6 +13534,7 @@ spec: }, }, "Name": "CloudquerySource-RemainingAwsDataContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -14369,6 +14391,7 @@ spec: }, }, "Name": "CloudquerySource-RiffRaffDataContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "RIFFRAFF_DB_USERNAME", @@ -14786,6 +14809,7 @@ spec: }, }, "Name": "CloudquerySource-SnykAllContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "SNYK_API_KEY", diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 6922ca605..b92a0e229 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -208,6 +208,7 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { App: app, Name: name, }, + readonlyRootFilesystem: true, command: [ '/bin/sh', '-c', From 02efd52ef91fa4d6e282941cfb3cc43cb9b18219 Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:43:48 +0100 Subject: [PATCH 02/21] write yaml to non-root folder --- .../service-catalogue.test.ts.snap | 144 +++++++++--------- packages/cdk/lib/cloudquery/task.ts | 8 +- 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 3d499555d..f0cf0041f 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -117,7 +117,7 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -141,7 +141,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -152,7 +152,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -766,7 +766,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -806,7 +806,7 @@ spec: value: RESOLVED severity_normalized: - Gte: 50 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -817,7 +817,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -1401,7 +1401,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -1424,7 +1424,7 @@ spec: accounts: - id: cq-for-000000000018 role_arn: arn:aws:iam::000000000018:role/cloudquery-access -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -1435,7 +1435,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -2019,7 +2019,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: fastly path: cloudquery/fastly @@ -2036,7 +2036,7 @@ spec: spec: concurrency: 1000 fastly_api_key: \${FASTLY_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2047,7 +2047,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -2629,7 +2629,7 @@ spec: "Fn::Join": [ "", [ - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: galaxies path: guardian/galaxies @@ -2668,7 +2668,7 @@ spec: ], }, " -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2679,7 +2679,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], ], }, @@ -3510,7 +3510,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Dumping yaml config";printf 'kind: source spec: name: github path: cloudquery/github @@ -3528,7 +3528,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3539,7 +3539,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -3961,7 +3961,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: github-languages path: guardian/github-languages @@ -3971,7 +3971,7 @@ spec: tables: - github_languages registry: github -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3982,7 +3982,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -4552,7 +4552,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Dumping yaml config";printf 'kind: source spec: name: github path: cloudquery/github @@ -4577,7 +4577,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -4588,7 +4588,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -5224,7 +5224,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Dumping yaml config";printf 'kind: source spec: name: github path: cloudquery/github @@ -5249,7 +5249,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5260,7 +5260,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -5896,7 +5896,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: ns1 registry: grpc @@ -5908,7 +5908,7 @@ spec: - postgresql spec: apiKey: \${NS1_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5919,7 +5919,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -6549,7 +6549,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -6573,7 +6573,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -6584,7 +6584,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -7168,7 +7168,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7194,7 +7194,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7205,7 +7205,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -7789,7 +7789,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7813,7 +7813,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7824,7 +7824,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -8438,7 +8438,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -8462,7 +8462,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -8473,7 +8473,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -9261,7 +9261,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9285,7 +9285,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9296,7 +9296,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -9646,7 +9646,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9670,7 +9670,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9681,7 +9681,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -10265,7 +10265,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10291,7 +10291,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10302,7 +10302,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -10916,7 +10916,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10941,7 +10941,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10952,7 +10952,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -11770,7 +11770,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -11795,7 +11795,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -11806,7 +11806,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -12390,7 +12390,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12417,7 +12417,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12428,7 +12428,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -12778,7 +12778,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12802,7 +12802,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12813,7 +12813,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -13397,7 +13397,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -13484,7 +13484,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -13495,7 +13495,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -14326,7 +14326,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: postgresql path: cloudquery/postgresql @@ -14341,7 +14341,7 @@ spec: connection_string: >- user=\${RIFFRAFF_DB_USERNAME} password=\${RIFFRAFF_DB_PASSWORD} host=\${RIFFRAFF_DB_HOST} port=5432 dbname=riffraff sslmode=verify-full -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14352,7 +14352,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { @@ -14744,7 +14744,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Dumping yaml config";printf 'kind: source spec: name: snyk path: cloudquery/snyk @@ -14759,7 +14759,7 @@ spec: - postgresql spec: api_key: \${SNYK_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /app/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14770,7 +14770,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", ], "DependsOn": [ { diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index b92a0e229..e70deea7e 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -219,11 +219,11 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { Install the CA bundle for all RDS certificates. See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html#UsingWithRDS.SSL.CertificatesAllRegions */ + 'echo "Dumping yaml config"', + `printf '${dump(sourceConfig)}' > /app/source.yaml`, + `printf '${dump(destinationConfig)}' > /app/destination.yaml`, + '/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console', 'wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates', - - `printf '${dump(sourceConfig)}' > /source.yaml`, - `printf '${dump(destinationConfig)}' > /destination.yaml`, - '/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console', ].join(';'), ], logging: fireLensLogDriver, From af3ce313e37b2ee1fc7aafb17dc157eb0c46061b Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:49:00 +0100 Subject: [PATCH 03/21] Revert "write yaml to non-root folder" This reverts commit 2970c643f4a7aad2d9fb50d7d4646afe502e139d. --- .../service-catalogue.test.ts.snap | 144 +++++++++--------- packages/cdk/lib/cloudquery/task.ts | 8 +- 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index f0cf0041f..3d499555d 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -117,7 +117,7 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -141,7 +141,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -152,7 +152,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -766,7 +766,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -806,7 +806,7 @@ spec: value: RESOLVED severity_normalized: - Gte: 50 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -817,7 +817,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -1401,7 +1401,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -1424,7 +1424,7 @@ spec: accounts: - id: cq-for-000000000018 role_arn: arn:aws:iam::000000000018:role/cloudquery-access -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -1435,7 +1435,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -2019,7 +2019,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: fastly path: cloudquery/fastly @@ -2036,7 +2036,7 @@ spec: spec: concurrency: 1000 fastly_api_key: \${FASTLY_API_KEY} -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2047,7 +2047,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -2629,7 +2629,7 @@ spec: "Fn::Join": [ "", [ - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: galaxies path: guardian/galaxies @@ -2668,7 +2668,7 @@ spec: ], }, " -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2679,7 +2679,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], ], }, @@ -3510,7 +3510,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Dumping yaml config";printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: github path: cloudquery/github @@ -3528,7 +3528,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3539,7 +3539,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -3961,7 +3961,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: github-languages path: guardian/github-languages @@ -3971,7 +3971,7 @@ spec: tables: - github_languages registry: github -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3982,7 +3982,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -4552,7 +4552,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Dumping yaml config";printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: github path: cloudquery/github @@ -4577,7 +4577,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -4588,7 +4588,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -5224,7 +5224,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Dumping yaml config";printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: github path: cloudquery/github @@ -5249,7 +5249,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5260,7 +5260,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -5896,7 +5896,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: ns1 registry: grpc @@ -5908,7 +5908,7 @@ spec: - postgresql spec: apiKey: \${NS1_API_KEY} -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5919,7 +5919,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -6549,7 +6549,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -6573,7 +6573,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -6584,7 +6584,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -7168,7 +7168,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7194,7 +7194,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7205,7 +7205,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -7789,7 +7789,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7813,7 +7813,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7824,7 +7824,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -8438,7 +8438,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -8462,7 +8462,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -8473,7 +8473,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -9261,7 +9261,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9285,7 +9285,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9296,7 +9296,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -9646,7 +9646,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9670,7 +9670,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9681,7 +9681,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -10265,7 +10265,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10291,7 +10291,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10302,7 +10302,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -10916,7 +10916,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10941,7 +10941,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10952,7 +10952,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -11770,7 +11770,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -11795,7 +11795,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -11806,7 +11806,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -12390,7 +12390,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12417,7 +12417,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12428,7 +12428,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -12778,7 +12778,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12802,7 +12802,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12813,7 +12813,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -13397,7 +13397,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: aws path: cloudquery/aws @@ -13484,7 +13484,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -13495,7 +13495,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -14326,7 +14326,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: postgresql path: cloudquery/postgresql @@ -14341,7 +14341,7 @@ spec: connection_string: >- user=\${RIFFRAFF_DB_USERNAME} password=\${RIFFRAFF_DB_PASSWORD} host=\${RIFFRAFF_DB_HOST} port=5432 dbname=riffraff sslmode=verify-full -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14352,7 +14352,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -14744,7 +14744,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Dumping yaml config";printf 'kind: source + "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source spec: name: snyk path: cloudquery/snyk @@ -14759,7 +14759,7 @@ spec: - postgresql spec: api_key: \${SNYK_API_KEY} -' > /app/source.yaml;printf 'kind: destination +' > /source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14770,7 +14770,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /app/destination.yaml;/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates", +' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", ], "DependsOn": [ { diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index e70deea7e..b92a0e229 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -219,11 +219,11 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { Install the CA bundle for all RDS certificates. See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html#UsingWithRDS.SSL.CertificatesAllRegions */ - 'echo "Dumping yaml config"', - `printf '${dump(sourceConfig)}' > /app/source.yaml`, - `printf '${dump(destinationConfig)}' > /app/destination.yaml`, - '/app/cloudquery sync /app/source.yaml app/destination.yaml --log-format json --log-console', 'wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates', + + `printf '${dump(sourceConfig)}' > /source.yaml`, + `printf '${dump(destinationConfig)}' > /destination.yaml`, + '/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console', ].join(';'), ], logging: fireLensLogDriver, From ca30825d929c7eeb1825894dfc1505009777fece Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:02:53 +0100 Subject: [PATCH 04/21] Create mount point for writeable directory Co-authored-by: Ashleigh Carr --- .../service-catalogue.test.ts.snap | 432 +++++++++++++++--- packages/cdk/lib/cloudquery/task.ts | 24 +- 2 files changed, 379 insertions(+), 77 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 3d499555d..4a7afbcd9 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -117,7 +117,7 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -141,7 +141,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -152,7 +152,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -190,6 +190,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-AwsCostExplorerContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -443,6 +450,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -766,7 +778,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -806,7 +818,7 @@ spec: value: RESOLVED severity_normalized: - Gte: 50 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -817,7 +829,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -855,6 +867,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-DelegatedToSecurityAccountContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -1108,6 +1127,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -1401,7 +1425,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -1424,7 +1448,7 @@ spec: accounts: - id: cq-for-000000000018 role_arn: arn:aws:iam::000000000018:role/cloudquery-access -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -1435,7 +1459,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -1473,6 +1497,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-DeployToolsListOrgsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -1726,6 +1757,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -2019,7 +2055,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: fastly path: cloudquery/fastly @@ -2036,7 +2072,7 @@ spec: spec: concurrency: 1000 fastly_api_key: \${FASTLY_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2047,7 +2083,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -2085,6 +2121,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-FastlyServicesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -2352,6 +2395,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -2629,7 +2677,7 @@ spec: "Fn::Join": [ "", [ - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: galaxies path: guardian/galaxies @@ -2668,7 +2716,7 @@ spec: ], }, " -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2679,7 +2727,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], ], }, @@ -2720,6 +2768,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-GalaxiesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -2973,6 +3028,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -3510,7 +3570,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Getting yaml config";printf 'kind: source spec: name: github path: cloudquery/github @@ -3528,7 +3588,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3539,7 +3599,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -3577,6 +3637,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-GitHubIssuesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -3872,6 +3939,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -3961,7 +4033,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: github-languages path: guardian/github-languages @@ -3971,7 +4043,7 @@ spec: tables: - github_languages registry: github -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3982,7 +4054,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -4020,6 +4092,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-GitHubLanguagesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -4279,6 +4358,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -4552,7 +4636,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Getting yaml config";printf 'kind: source spec: name: github path: cloudquery/github @@ -4577,7 +4661,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -4588,7 +4672,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -4626,6 +4710,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-GitHubRepositoriesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -4921,6 +5012,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -5224,7 +5320,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Getting yaml config";printf 'kind: source spec: name: github path: cloudquery/github @@ -5249,7 +5345,7 @@ spec: private_key_path: /github-private-key app_id: \${file:/github-app-id} installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5260,7 +5356,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -5298,6 +5394,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-GitHubTeamsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -5593,6 +5696,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -5896,7 +6004,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: ns1 registry: grpc @@ -5908,7 +6016,7 @@ spec: - postgresql spec: apiKey: \${NS1_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5919,7 +6027,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -5961,6 +6069,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-NS1Container", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -6246,6 +6361,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -6549,7 +6669,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -6573,7 +6693,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -6584,7 +6704,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -6622,6 +6742,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideAutoScalingGroupsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -6875,6 +7002,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -7168,7 +7300,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7194,7 +7326,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7205,7 +7337,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -7243,6 +7375,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideBackupContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -7496,6 +7635,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -7789,7 +7933,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7813,7 +7957,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7824,7 +7968,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -7862,6 +8006,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideCertificatesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -8115,6 +8266,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -8438,7 +8594,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -8462,7 +8618,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -8473,7 +8629,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -8511,6 +8667,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideCloudFormationContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -8764,6 +8927,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -9261,7 +9429,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9285,7 +9453,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9296,7 +9464,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -9334,6 +9502,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideCloudwatchAlarmsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -9587,6 +9762,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -9646,7 +9826,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9670,7 +9850,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9681,7 +9861,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -9719,6 +9899,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideDynamoDBContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -9972,6 +10159,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -10265,7 +10457,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10291,7 +10483,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10302,7 +10494,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -10344,6 +10536,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideEc2Container", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -10623,6 +10822,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -10916,7 +11120,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10941,7 +11145,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10952,7 +11156,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -10990,6 +11194,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideInspectorContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -11243,6 +11454,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -11770,7 +11986,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -11795,7 +12011,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -11806,7 +12022,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -11844,6 +12060,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideLoadBalancersContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -12097,6 +12320,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -12390,7 +12618,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12417,7 +12645,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12428,7 +12656,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -12466,6 +12694,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideRDSContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -12719,6 +12954,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -12778,7 +13018,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12802,7 +13042,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12813,7 +13053,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -12851,6 +13091,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-OrgWideS3Container", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -13104,6 +13351,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -13397,7 +13649,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: aws path: cloudquery/aws @@ -13484,7 +13736,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -13495,7 +13747,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -13533,6 +13785,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-RemainingAwsDataContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -13786,6 +14045,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -14326,7 +14590,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: postgresql path: cloudquery/postgresql @@ -14341,7 +14605,7 @@ spec: connection_string: >- user=\${RIFFRAFF_DB_USERNAME} password=\${RIFFRAFF_DB_PASSWORD} host=\${RIFFRAFF_DB_HOST} port=5432 dbname=riffraff sslmode=verify-full -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14352,7 +14616,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -14390,6 +14654,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-RiffRaffDataContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -14685,6 +14956,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -14744,7 +15020,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo "Getting yaml config";printf 'kind: source spec: name: snyk path: cloudquery/snyk @@ -14759,7 +15035,7 @@ spec: - postgresql spec: api_key: \${SNYK_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /data/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14770,7 +15046,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", ], "DependsOn": [ { @@ -14808,6 +15084,13 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/data", + "ReadOnly": false, + "SourceVolume": "data", + }, + ], "Name": "CloudquerySource-SnykAllContainer", "ReadonlyRootFilesystem": true, "Secrets": [ @@ -15075,6 +15358,11 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "data", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index b92a0e229..8b30f0ccd 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -189,6 +189,8 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { }, }); + const volumePath = '/data'; + const cloudqueryTask = task.addContainer(`${id}Container`, { image: Images.cloudquery, entryPoint: [''], @@ -214,21 +216,33 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { '-c', [ ...additionalCommands, - + 'echo "Getting yaml config"', + `printf '${dump(sourceConfig)}' > ${volumePath}/source.yaml`, + `printf '${dump(destinationConfig)}' > ${volumePath}/destination.yaml`, + 'echo "Got yaml config"', /* Install the CA bundle for all RDS certificates. See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html#UsingWithRDS.SSL.CertificatesAllRegions */ 'wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates', - - `printf '${dump(sourceConfig)}' > /source.yaml`, - `printf '${dump(destinationConfig)}' > /destination.yaml`, - '/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console', + `/app/cloudquery sync ${volumePath}/source.yaml ${volumePath}/destination.yaml --log-format json --log-console`, ].join(';'), ], logging: fireLensLogDriver, }); + const mountPoint = { + containerPath: volumePath, + sourceVolume: 'data', + readOnly: false, + }; + + task.addVolume({ + name: 'data', + }); + + cloudqueryTask.addMountPoints(mountPoint); + const otel = task.addContainer(`${id}AWSOTELCollector`, { image: Images.otelCollector, command: ['--config=/etc/ecs/ecs-xray.yaml'], From fd18e96ff67c55cd08131787892a3251ebc4c158 Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Tue, 16 Apr 2024 16:16:50 +0100 Subject: [PATCH 05/21] Don't install RDS CA Co-authored-by: Natasha <67543397+NovemberTang@users.noreply.github.com> --- .../service-catalogue.test.ts.snap | 48 +++++++++---------- packages/cdk/lib/cloudquery/task.ts | 7 +-- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 4a7afbcd9..1702da19d 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -152,7 +152,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -829,7 +829,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -1459,7 +1459,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -2083,7 +2083,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -2727,7 +2727,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], ], }, @@ -3599,7 +3599,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4054,7 +4054,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4672,7 +4672,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -5356,7 +5356,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -6027,7 +6027,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -6704,7 +6704,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -7337,7 +7337,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -7968,7 +7968,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -8629,7 +8629,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -9464,7 +9464,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -9861,7 +9861,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -10494,7 +10494,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -11156,7 +11156,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -12022,7 +12022,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -12656,7 +12656,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -13053,7 +13053,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -13747,7 +13747,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -14616,7 +14616,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -15046,7 +15046,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console", +' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 8b30f0ccd..6826e6df6 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -220,12 +220,7 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { `printf '${dump(sourceConfig)}' > ${volumePath}/source.yaml`, `printf '${dump(destinationConfig)}' > ${volumePath}/destination.yaml`, 'echo "Got yaml config"', - /* - Install the CA bundle for all RDS certificates. - See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html#UsingWithRDS.SSL.CertificatesAllRegions - */ - 'wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates', - `/app/cloudquery sync ${volumePath}/source.yaml ${volumePath}/destination.yaml --log-format json --log-console`, + `/app/cloudquery sync ${volumePath}/source.yaml ${volumePath}/destination.yaml --log-format json --log-console --no-log-file`, ].join(';'), ], logging: fireLensLogDriver, From 6b8b8bcfbde67f18aefbafde3738587168043f63 Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Tue, 16 Apr 2024 16:40:07 +0100 Subject: [PATCH 06/21] Add a volume to container Cloudquery internal data Co-authored-by: Natasha <67543397+NovemberTang@users.noreply.github.com> --- .../service-catalogue.test.ts.snap | 288 +++++++++++++++--- packages/cdk/lib/cloudquery/task.ts | 26 +- 2 files changed, 258 insertions(+), 56 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 1702da19d..dee0f2f44 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -194,7 +194,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-AwsCostExplorerContainer", @@ -452,7 +457,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -871,7 +879,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-DelegatedToSecurityAccountContainer", @@ -1129,7 +1142,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -1501,7 +1517,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-DeployToolsListOrgsContainer", @@ -1759,7 +1780,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -2125,7 +2149,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-FastlyServicesContainer", @@ -2397,7 +2426,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -2772,7 +2804,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-GalaxiesContainer", @@ -3030,7 +3067,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -3641,7 +3681,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-GitHubIssuesContainer", @@ -3941,7 +3986,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -4096,7 +4144,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-GitHubLanguagesContainer", @@ -4360,7 +4413,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -4714,7 +4770,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-GitHubRepositoriesContainer", @@ -5014,7 +5075,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -5398,7 +5462,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-GitHubTeamsContainer", @@ -5698,7 +5767,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -6073,7 +6145,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-NS1Container", @@ -6363,7 +6440,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -6746,7 +6826,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideAutoScalingGroupsContainer", @@ -7004,7 +7089,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -7379,7 +7467,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideBackupContainer", @@ -7637,7 +7730,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -8010,7 +8106,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideCertificatesContainer", @@ -8268,7 +8369,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -8671,7 +8775,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideCloudFormationContainer", @@ -8929,7 +9038,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -9506,7 +9618,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideCloudwatchAlarmsContainer", @@ -9764,7 +9881,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -9903,7 +10023,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideDynamoDBContainer", @@ -10161,7 +10286,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -10540,7 +10668,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideEc2Container", @@ -10824,7 +10957,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -11198,7 +11334,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideInspectorContainer", @@ -11456,7 +11597,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -12064,7 +12208,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideLoadBalancersContainer", @@ -12322,7 +12471,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -12698,7 +12850,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideRDSContainer", @@ -12956,7 +13113,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -13095,7 +13255,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-OrgWideS3Container", @@ -13353,7 +13518,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -13789,7 +13957,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-RemainingAwsDataContainer", @@ -14047,7 +14220,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -14658,7 +14834,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-RiffRaffDataContainer", @@ -14958,7 +15139,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, @@ -15088,7 +15272,12 @@ spec: { "ContainerPath": "/data", "ReadOnly": false, - "SourceVolume": "data", + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", }, ], "Name": "CloudquerySource-SnykAllContainer", @@ -15360,7 +15549,10 @@ spec: }, "Volumes": [ { - "Name": "data", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", }, ], }, diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 6826e6df6..74d30e511 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -226,17 +226,27 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { logging: fireLensLogDriver, }); - const mountPoint = { - containerPath: volumePath, - sourceVolume: 'data', - readOnly: false, - }; - task.addVolume({ - name: 'data', + name: 'config-volume', + }); + task.addVolume({ + name: 'cloudquery-volume', }); - cloudqueryTask.addMountPoints(mountPoint); + cloudqueryTask.addMountPoints( + { + // So that we can write task config to this directory + containerPath: volumePath, + sourceVolume: 'config-volume', + readOnly: false, + }, + { + // So that Cloudquery can write to this directory + containerPath: '/app/.cq', + sourceVolume: 'cloudquery-volume', + readOnly: false, + }, + ); const otel = task.addContainer(`${id}AWSOTELCollector`, { image: Images.otelCollector, From 914d6f89dc74841d251155ceb8e7185ec81fa900 Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Tue, 16 Apr 2024 16:52:28 +0100 Subject: [PATCH 07/21] Add volume for /tmp folder Co-authored-by: Natasha <67543397+NovemberTang@users.noreply.github.com> --- .../service-catalogue.test.ts.snap | 192 ++++++++++++++++++ packages/cdk/lib/cloudquery/task.ts | 9 + 2 files changed, 201 insertions(+) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index dee0f2f44..162ff6139 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -201,6 +201,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-AwsCostExplorerContainer", "ReadonlyRootFilesystem": true, @@ -462,6 +467,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -886,6 +894,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-DelegatedToSecurityAccountContainer", "ReadonlyRootFilesystem": true, @@ -1147,6 +1160,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -1524,6 +1540,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-DeployToolsListOrgsContainer", "ReadonlyRootFilesystem": true, @@ -1785,6 +1806,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -2156,6 +2180,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-FastlyServicesContainer", "ReadonlyRootFilesystem": true, @@ -2431,6 +2460,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -2811,6 +2843,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-GalaxiesContainer", "ReadonlyRootFilesystem": true, @@ -3072,6 +3109,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -3688,6 +3728,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-GitHubIssuesContainer", "ReadonlyRootFilesystem": true, @@ -3991,6 +4036,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -4151,6 +4199,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-GitHubLanguagesContainer", "ReadonlyRootFilesystem": true, @@ -4418,6 +4471,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -4777,6 +4833,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-GitHubRepositoriesContainer", "ReadonlyRootFilesystem": true, @@ -5080,6 +5141,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -5469,6 +5533,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-GitHubTeamsContainer", "ReadonlyRootFilesystem": true, @@ -5772,6 +5841,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -6152,6 +6224,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-NS1Container", "ReadonlyRootFilesystem": true, @@ -6445,6 +6522,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -6833,6 +6913,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideAutoScalingGroupsContainer", "ReadonlyRootFilesystem": true, @@ -7094,6 +7179,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -7474,6 +7562,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideBackupContainer", "ReadonlyRootFilesystem": true, @@ -7735,6 +7828,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -8113,6 +8209,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideCertificatesContainer", "ReadonlyRootFilesystem": true, @@ -8374,6 +8475,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -8782,6 +8886,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideCloudFormationContainer", "ReadonlyRootFilesystem": true, @@ -9043,6 +9152,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -9625,6 +9737,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideCloudwatchAlarmsContainer", "ReadonlyRootFilesystem": true, @@ -9886,6 +10003,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -10030,6 +10150,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideDynamoDBContainer", "ReadonlyRootFilesystem": true, @@ -10291,6 +10416,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -10675,6 +10803,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideEc2Container", "ReadonlyRootFilesystem": true, @@ -10962,6 +11095,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -11341,6 +11477,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideInspectorContainer", "ReadonlyRootFilesystem": true, @@ -11602,6 +11743,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -12215,6 +12359,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideLoadBalancersContainer", "ReadonlyRootFilesystem": true, @@ -12476,6 +12625,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -12857,6 +13009,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideRDSContainer", "ReadonlyRootFilesystem": true, @@ -13118,6 +13275,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -13262,6 +13422,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-OrgWideS3Container", "ReadonlyRootFilesystem": true, @@ -13523,6 +13688,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -13964,6 +14132,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-RemainingAwsDataContainer", "ReadonlyRootFilesystem": true, @@ -14225,6 +14398,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -14841,6 +15017,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-RiffRaffDataContainer", "ReadonlyRootFilesystem": true, @@ -15144,6 +15325,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", @@ -15279,6 +15463,11 @@ spec: "ReadOnly": false, "SourceVolume": "cloudquery-volume", }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, ], "Name": "CloudquerySource-SnykAllContainer", "ReadonlyRootFilesystem": true, @@ -15554,6 +15743,9 @@ spec: { "Name": "cloudquery-volume", }, + { + "Name": "tmp-volume", + }, ], }, "Type": "AWS::ECS::TaskDefinition", diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 74d30e511..2273af11c 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -232,6 +232,9 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { task.addVolume({ name: 'cloudquery-volume', }); + task.addVolume({ + name: 'tmp-volume', + }); cloudqueryTask.addMountPoints( { @@ -246,6 +249,12 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { sourceVolume: 'cloudquery-volume', readOnly: false, }, + { + // So that Cloudquery can write temporary data + containerPath: '/tmp', + sourceVolume: 'tmp-volume', + readOnly: false, + }, ); const otel = task.addContainer(`${id}AWSOTELCollector`, { From c905ef7be1f768bd3de112b259fc72a22e992f32 Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Tue, 16 Apr 2024 17:04:42 +0100 Subject: [PATCH 08/21] Publish a new Cloudquery image with Amazon RDS CA Co-authored-by: Natasha <67543397+NovemberTang@users.noreply.github.com> --- .github/workflows/cloudquery-image.yml | 65 ++++++++++++++++++++++++++ containers/cloudquery/Dockerfile | 5 ++ 2 files changed, 70 insertions(+) create mode 100644 .github/workflows/cloudquery-image.yml create mode 100644 containers/cloudquery/Dockerfile diff --git a/.github/workflows/cloudquery-image.yml b/.github/workflows/cloudquery-image.yml new file mode 100644 index 000000000..998376f5c --- /dev/null +++ b/.github/workflows/cloudquery-image.yml @@ -0,0 +1,65 @@ +# Find full documentation here https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions +name: prisma-migrate image + +on: + pull_request: + paths: + - 'containers/prisma-migrate/**' + push: + branches: + - main + paths: + - 'containers/prisma-migrate/**' + + # Manual invocation. + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/prisma-migrate + +# Ensure we only ever have one build running at a time. +# If we push twice in quick succession, the first build will be stopped once the second starts. +# This avoids any race conditions. +concurrency: + group: ${{ github.ref }}/prisma-migrate + cancel-in-progress: true + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,format=long + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./ + file: containers/prisma-migrate/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/containers/cloudquery/Dockerfile b/containers/cloudquery/Dockerfile new file mode 100644 index 000000000..3f9db86be --- /dev/null +++ b/containers/cloudquery/Dockerfile @@ -0,0 +1,5 @@ +FROM ghcr.io/cloudquery/cloudquery:5.2.0 + +# Need to install RDS certs before running Cloudquery container due to +# access to the root filesystem being restricted +RUN wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates From 8ab5c3767bc4d0e4491ab974af9f9db0798d06dd Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:29:31 +0100 Subject: [PATCH 09/21] trigger first workflow run --- .../{cloudquery-image.yml => cq-image.yml} | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) rename .github/workflows/{cloudquery-image.yml => cq-image.yml} (83%) diff --git a/.github/workflows/cloudquery-image.yml b/.github/workflows/cq-image.yml similarity index 83% rename from .github/workflows/cloudquery-image.yml rename to .github/workflows/cq-image.yml index 998376f5c..d87d731d6 100644 --- a/.github/workflows/cloudquery-image.yml +++ b/.github/workflows/cq-image.yml @@ -1,28 +1,24 @@ # Find full documentation here https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions -name: prisma-migrate image +name: CQ image on: - pull_request: - paths: - - 'containers/prisma-migrate/**' push: branches: - main - paths: - - 'containers/prisma-migrate/**' + - nt/ecs-read-only # Manual invocation. workflow_dispatch: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}/prisma-migrate + IMAGE_NAME: ${{ github.repository }}/cloudquery # Ensure we only ever have one build running at a time. # If we push twice in quick succession, the first build will be stopped once the second starts. # This avoids any race conditions. concurrency: - group: ${{ github.ref }}/prisma-migrate + group: ${{ github.ref }}/cloudquery cancel-in-progress: true jobs: @@ -58,7 +54,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ./ - file: containers/prisma-migrate/Dockerfile + file: containers/cloudquery/Dockerfile platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} From fd2ca34d07bac3cd6bdd35e07dbd6ad76df0565f Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:49:45 +0100 Subject: [PATCH 10/21] only run on changes under particular paths --- .github/workflows/cq-image.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cq-image.yml b/.github/workflows/cq-image.yml index d87d731d6..012a13f91 100644 --- a/.github/workflows/cq-image.yml +++ b/.github/workflows/cq-image.yml @@ -2,10 +2,14 @@ name: CQ image on: + pull_request: + paths: + - 'containers/cloudquery/**' push: branches: - main - - nt/ecs-read-only + paths: + - 'containers/cloudquery/**' # Manual invocation. workflow_dispatch: From d44939caf9b9263c127a25b7834e9b6c8f168eee Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:54:55 +0100 Subject: [PATCH 11/21] use new container image --- .../service-catalogue.test.ts.snap | 48 +++++++++---------- packages/cdk/lib/cloudquery/images.ts | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 162ff6139..3e7460854 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -176,7 +176,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -869,7 +869,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -1515,7 +1515,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -2155,7 +2155,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -2818,7 +2818,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -3703,7 +3703,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -4174,7 +4174,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -4808,7 +4808,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -5508,7 +5508,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -6199,7 +6199,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -6888,7 +6888,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -7537,7 +7537,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -8184,7 +8184,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -8861,7 +8861,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -9712,7 +9712,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10125,7 +10125,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10778,7 +10778,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -11452,7 +11452,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -12334,7 +12334,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -12984,7 +12984,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -13397,7 +13397,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -14107,7 +14107,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -14992,7 +14992,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -15438,7 +15438,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { diff --git a/packages/cdk/lib/cloudquery/images.ts b/packages/cdk/lib/cloudquery/images.ts index 464323796..3c112b159 100644 --- a/packages/cdk/lib/cloudquery/images.ts +++ b/packages/cdk/lib/cloudquery/images.ts @@ -3,7 +3,7 @@ import { Versions } from './versions'; export const Images = { cloudquery: ContainerImage.fromRegistry( - `ghcr.io/cloudquery/cloudquery:${Versions.CloudqueryCli}`, + `ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066`, ), devxLogs: ContainerImage.fromRegistry('ghcr.io/guardian/devx-logs:2'), amazonLinux: ContainerImage.fromRegistry( From 0cb1fef4068810d9fe85abc48718582bb0a94d0d Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:12:52 +0100 Subject: [PATCH 12/21] load cloudquery version from .env --- .github/workflows/cq-image.yml | 7 +++++++ containers/cloudquery/Dockerfile | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cq-image.yml b/.github/workflows/cq-image.yml index 012a13f91..7100c3b1d 100644 --- a/.github/workflows/cq-image.yml +++ b/.github/workflows/cq-image.yml @@ -54,6 +54,12 @@ jobs: tags: | type=sha,format=long + - name: Load .env file + uses: xom9ikk/dotenv@ac290ca23a42155a0cba1031d23afa46240116a9 # v2.3.0 + + - name: Echo CQ_VERSION + run: echo ${{ env.CQ_CLI }} + - name: Build and push Docker image uses: docker/build-push-action@v5 with: @@ -63,3 +69,4 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: CQ_CLI=${{ env.CQ_CLI }} diff --git a/containers/cloudquery/Dockerfile b/containers/cloudquery/Dockerfile index 3f9db86be..6ef294756 100644 --- a/containers/cloudquery/Dockerfile +++ b/containers/cloudquery/Dockerfile @@ -1,4 +1,6 @@ -FROM ghcr.io/cloudquery/cloudquery:5.2.0 +ARG CQ_CLI + +FROM ghcr.io/cloudquery/cloudquery:${CQ_CLI} # Need to install RDS certs before running Cloudquery container due to # access to the root filesystem being restricted From c95a3209d27e964bdd1cc8854b8faac7175b3b0f Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:37:32 +0100 Subject: [PATCH 13/21] refer to new image --- .../service-catalogue.test.ts.snap | 48 +++++++++---------- packages/cdk/lib/cloudquery/images.ts | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 3e7460854..16da6a280 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -176,7 +176,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -869,7 +869,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -1515,7 +1515,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -2155,7 +2155,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -2818,7 +2818,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -3703,7 +3703,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -4174,7 +4174,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -4808,7 +4808,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -5508,7 +5508,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -6199,7 +6199,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -6888,7 +6888,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -7537,7 +7537,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -8184,7 +8184,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -8861,7 +8861,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -9712,7 +9712,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10125,7 +10125,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10778,7 +10778,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -11452,7 +11452,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -12334,7 +12334,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -12984,7 +12984,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -13397,7 +13397,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -14107,7 +14107,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -14992,7 +14992,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -15438,7 +15438,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { diff --git a/packages/cdk/lib/cloudquery/images.ts b/packages/cdk/lib/cloudquery/images.ts index 3c112b159..043adb86a 100644 --- a/packages/cdk/lib/cloudquery/images.ts +++ b/packages/cdk/lib/cloudquery/images.ts @@ -3,7 +3,7 @@ import { Versions } from './versions'; export const Images = { cloudquery: ContainerImage.fromRegistry( - `ghcr.io/guardian/service-catalogue/cloudquery:sha-bacccb49c85d2b461bb5fad1be84c534e476a066`, + `ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e`, ), devxLogs: ContainerImage.fromRegistry('ghcr.io/guardian/devx-logs:2'), amazonLinux: ContainerImage.fromRegistry( From 8de7567bfaf119c86a280b3655c8d456954223d8 Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:33:40 +0100 Subject: [PATCH 14/21] remove log lines --- .../service-catalogue.test.ts.snap | 96 +++++++++---------- packages/cdk/lib/cloudquery/task.ts | 2 - 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 16da6a280..0d12dec4f 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -117,7 +117,7 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -152,7 +152,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -794,7 +794,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -845,7 +845,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -1457,7 +1457,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -1491,7 +1491,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -2103,7 +2103,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: fastly path: cloudquery/fastly @@ -2131,7 +2131,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -2741,7 +2741,7 @@ spec: "Fn::Join": [ "", [ - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: galaxies path: guardian/galaxies @@ -2791,7 +2791,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], ], }, @@ -3650,7 +3650,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Getting yaml config";printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -3679,7 +3679,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4129,7 +4129,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: github-languages path: guardian/github-languages @@ -4150,7 +4150,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4748,7 +4748,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Getting yaml config";printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -4784,7 +4784,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -5448,7 +5448,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;echo "Getting yaml config";printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -5484,7 +5484,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -6148,7 +6148,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: ns1 registry: grpc @@ -6171,7 +6171,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -6829,7 +6829,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -6864,7 +6864,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -7476,7 +7476,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7513,7 +7513,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -8125,7 +8125,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -8160,7 +8160,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -8802,7 +8802,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -8837,7 +8837,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -9653,7 +9653,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9688,7 +9688,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -10066,7 +10066,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10101,7 +10101,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -10713,7 +10713,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10750,7 +10750,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -11392,7 +11392,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -11428,7 +11428,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -12274,7 +12274,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12310,7 +12310,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -12922,7 +12922,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12960,7 +12960,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -13338,7 +13338,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -13373,7 +13373,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -13985,7 +13985,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -14083,7 +14083,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -14942,7 +14942,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: postgresql path: cloudquery/postgresql @@ -14968,7 +14968,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -15388,7 +15388,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo "Getting yaml config";printf 'kind: source + "printf 'kind: source spec: name: snyk path: cloudquery/snyk @@ -15414,7 +15414,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;echo "Got yaml config";/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 2273af11c..195416595 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -216,10 +216,8 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { '-c', [ ...additionalCommands, - 'echo "Getting yaml config"', `printf '${dump(sourceConfig)}' > ${volumePath}/source.yaml`, `printf '${dump(destinationConfig)}' > ${volumePath}/destination.yaml`, - 'echo "Got yaml config"', `/app/cloudquery sync ${volumePath}/source.yaml ${volumePath}/destination.yaml --log-format json --log-console --no-log-file`, ].join(';'), ], From 5a27510a31b99c47de28c7beebd3db58fbba46ae Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:23:03 +0100 Subject: [PATCH 15/21] move github cresentials to ephemeral storage --- .../service-catalogue.test.ts.snap | 24 +++++++++---------- packages/cdk/lib/cloudquery/config.test.ts | 6 ++--- packages/cdk/lib/cloudquery/config.ts | 7 +++--- packages/cdk/lib/cloudquery/index.ts | 6 ++--- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 0d12dec4f..4adf68775 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -3650,7 +3650,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key;echo -n $GITHUB_APP_ID > /data/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -3665,9 +3665,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} + private_key_path: /data/github-private-key + app_id: \${file:/data/github-app-id} + installation_id: \${file:/data/github-installation-id} ' > /data/source.yaml;printf 'kind: destination spec: name: postgresql @@ -4748,7 +4748,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key;echo -n $GITHUB_APP_ID > /data/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -4770,9 +4770,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} + private_key_path: /data/github-private-key + app_id: \${file:/data/github-app-id} + installation_id: \${file:/data/github-installation-id} ' > /data/source.yaml;printf 'kind: destination spec: name: postgresql @@ -5448,7 +5448,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key;echo -n $GITHUB_APP_ID > /data/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -5470,9 +5470,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} + private_key_path: /data/github-private-key + app_id: \${file:/data/github-app-id} + installation_id: \${file:/data/github-installation-id} ' > /data/source.yaml;printf 'kind: destination spec: name: postgresql diff --git a/packages/cdk/lib/cloudquery/config.test.ts b/packages/cdk/lib/cloudquery/config.test.ts index be139b52e..a134a19d7 100644 --- a/packages/cdk/lib/cloudquery/config.test.ts +++ b/packages/cdk/lib/cloudquery/config.test.ts @@ -196,9 +196,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} + private_key_path: /data/github-private-key + app_id: \${file:/data/github-app-id} + installation_id: \${file:/data/github-installation-id} " `); }); diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index 105275a36..d08a54a97 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -142,9 +142,10 @@ export function githubSourceConfig( org: 'guardian', // For simplicity, read all configuration from disk. - private_key_path: '/github-private-key', - app_id: '${file:/github-app-id}', - installation_id: '${file:/github-installation-id}', + //These values are duplicated. Can we use a variable for them? + private_key_path: '/data/github-private-key', + app_id: '${file:/data/github-app-id}', + installation_id: '${file:/data/github-installation-id}', }, ], }, diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index 9a9bdc64b..6bdff197d 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -341,9 +341,9 @@ export function addCloudqueryEcsCluster( }; const additionalGithubCommands = [ - 'echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key', - 'echo -n $GITHUB_APP_ID > /github-app-id', - 'echo -n $GITHUB_INSTALLATION_ID > /github-installation-id', + 'echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key', + 'echo -n $GITHUB_APP_ID > /data/github-app-id', + 'echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id', ]; const githubSources: CloudquerySource[] = [ From 4013d44bab05405aa679723667506dd517f8f22d Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Wed, 17 Apr 2024 12:45:13 +0100 Subject: [PATCH 16/21] Add a comment to additionalCommands instructing users to write to the `/data` folder Co-authored-by: Natasha <67543397+NovemberTang@users.noreply.github.com> --- packages/cdk/lib/cloudquery/task.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 195416595..86a14498b 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -82,6 +82,8 @@ export interface ScheduledCloudqueryTaskProps /** * Any additional commands to run within the CloudQuery container. * These are executed first. + * + * The containers filesystem is mostly read-only. If you need to write files you can use the /data folder. */ additionalCommands?: string[]; From abbb02a998d2d6376e66e4d4cc70750bb721ff3c Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Wed, 17 Apr 2024 12:59:13 +0100 Subject: [PATCH 17/21] Don't use external action for setting CQ_CLI env variable --- .github/workflows/cq-image.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cq-image.yml b/.github/workflows/cq-image.yml index 7100c3b1d..9d3f07d77 100644 --- a/.github/workflows/cq-image.yml +++ b/.github/workflows/cq-image.yml @@ -54,11 +54,10 @@ jobs: tags: | type=sha,format=long - - name: Load .env file - uses: xom9ikk/dotenv@ac290ca23a42155a0cba1031d23afa46240116a9 # v2.3.0 - - - name: Echo CQ_VERSION - run: echo ${{ env.CQ_CLI }} + - name: Set CQ_CLI Github Env variable from .env + run: | + source .env + echo "CQ_CLI=${CQ_CLI}" >> "$GITHUB_ENV" - name: Build and push Docker image uses: docker/build-push-action@v5 From 0218f848dff5eab2eed17669ea3b6ae204d07468 Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Wed, 17 Apr 2024 14:38:56 +0100 Subject: [PATCH 18/21] Use Standard linux folder for storing config --- .../service-catalogue.test.ts.snap | 150 +++++++++--------- packages/cdk/lib/cloudquery/config.ts | 2 + packages/cdk/lib/cloudquery/index.ts | 7 +- packages/cdk/lib/cloudquery/task.ts | 17 +- 4 files changed, 90 insertions(+), 86 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 4adf68775..7f010f5df 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -141,7 +141,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -152,7 +152,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -192,7 +192,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -834,7 +834,7 @@ spec: value: RESOLVED severity_normalized: - Gte: 50 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -845,7 +845,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -885,7 +885,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -1480,7 +1480,7 @@ spec: accounts: - id: cq-for-000000000018 role_arn: arn:aws:iam::000000000018:role/cloudquery-access -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -1491,7 +1491,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -1531,7 +1531,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -2120,7 +2120,7 @@ spec: spec: concurrency: 1000 fastly_api_key: \${FASTLY_API_KEY} -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2131,7 +2131,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -2171,7 +2171,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -2780,7 +2780,7 @@ spec: ], }, " -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2791,7 +2791,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], ], }, @@ -2834,7 +2834,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -3650,7 +3650,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key;echo -n $GITHUB_APP_ID > /data/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -3668,7 +3668,7 @@ spec: private_key_path: /data/github-private-key app_id: \${file:/data/github-app-id} installation_id: \${file:/data/github-installation-id} -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3679,7 +3679,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -3719,7 +3719,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -4139,7 +4139,7 @@ spec: tables: - github_languages registry: github -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -4150,7 +4150,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4190,7 +4190,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -4748,7 +4748,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key;echo -n $GITHUB_APP_ID > /data/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -4773,7 +4773,7 @@ spec: private_key_path: /data/github-private-key app_id: \${file:/data/github-app-id} installation_id: \${file:/data/github-installation-id} -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -4784,7 +4784,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4824,7 +4824,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -5448,7 +5448,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key;echo -n $GITHUB_APP_ID > /data/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -5473,7 +5473,7 @@ spec: private_key_path: /data/github-private-key app_id: \${file:/data/github-app-id} installation_id: \${file:/data/github-installation-id} -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5484,7 +5484,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -5524,7 +5524,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -6160,7 +6160,7 @@ spec: - postgresql spec: apiKey: \${NS1_API_KEY} -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -6171,7 +6171,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -6215,7 +6215,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -6853,7 +6853,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -6864,7 +6864,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -6904,7 +6904,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -7502,7 +7502,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7513,7 +7513,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -7553,7 +7553,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -8149,7 +8149,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -8160,7 +8160,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -8200,7 +8200,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -8826,7 +8826,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -8837,7 +8837,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -8877,7 +8877,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -9677,7 +9677,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9688,7 +9688,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -9728,7 +9728,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -10090,7 +10090,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10101,7 +10101,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -10141,7 +10141,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -10739,7 +10739,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10750,7 +10750,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -10794,7 +10794,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -11417,7 +11417,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -11428,7 +11428,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -11468,7 +11468,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -12299,7 +12299,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12310,7 +12310,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -12350,7 +12350,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -12949,7 +12949,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12960,7 +12960,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -13000,7 +13000,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -13362,7 +13362,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -13373,7 +13373,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -13413,7 +13413,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -14072,7 +14072,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14083,7 +14083,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -14123,7 +14123,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -14957,7 +14957,7 @@ spec: connection_string: >- user=\${RIFFRAFF_DB_USERNAME} password=\${RIFFRAFF_DB_PASSWORD} host=\${RIFFRAFF_DB_HOST} port=5432 dbname=riffraff sslmode=verify-full -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14968,7 +14968,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -15008,7 +15008,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, @@ -15403,7 +15403,7 @@ spec: - postgresql spec: api_key: \${SNYK_API_KEY} -' > /data/source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -15414,7 +15414,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /data/destination.yaml;/app/cloudquery sync /data/source.yaml /data/destination.yaml --log-format json --log-console --no-log-file", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -15454,7 +15454,7 @@ spec: }, "MountPoints": [ { - "ContainerPath": "/data", + "ContainerPath": "/usr/share/cloudquery", "ReadOnly": false, "SourceVolume": "config-volume", }, diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index d08a54a97..159bf1ebf 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -338,3 +338,5 @@ export const skipTables = [ 'aws_stepfunctions_map_run_executions', 'aws_stepfunctions_executions', ]; + +export const serviceCatalogueConfigDirectory = '/usr/share/cloudquery'; diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index 6bdff197d..f04048d30 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -22,6 +22,7 @@ import { githubSourceConfig, ns1SourceConfig, riffraffSourcesConfig, + serviceCatalogueConfigDirectory, skipTables, snykSourceConfig, } from './config'; @@ -341,9 +342,9 @@ export function addCloudqueryEcsCluster( }; const additionalGithubCommands = [ - 'echo -n $GITHUB_PRIVATE_KEY | base64 -d > /data/github-private-key', - 'echo -n $GITHUB_APP_ID > /data/github-app-id', - 'echo -n $GITHUB_INSTALLATION_ID > /data/github-installation-id', + `echo -n $GITHUB_PRIVATE_KEY | base64 -d > ${serviceCatalogueConfigDirectory}/github-private-key`, + `echo -n $GITHUB_APP_ID > ${serviceCatalogueConfigDirectory}/github-app-id`, + `echo -n $GITHUB_INSTALLATION_ID > ${serviceCatalogueConfigDirectory}/github-installation-id`, ]; const githubSources: CloudquerySource[] = [ diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 86a14498b..cf0fe1267 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -20,7 +20,10 @@ import { RetentionDays } from 'aws-cdk-lib/aws-logs'; import type { DatabaseInstance } from 'aws-cdk-lib/aws-rds'; import { dump } from 'js-yaml'; import type { CloudqueryConfig } from './config'; -import { postgresDestinationConfig } from './config'; +import { + postgresDestinationConfig, + serviceCatalogueConfigDirectory, +} from './config'; import { Images } from './images'; import { singletonPolicy } from './policies'; import { scheduleFrequency } from './schedule'; @@ -83,7 +86,7 @@ export interface ScheduledCloudqueryTaskProps * Any additional commands to run within the CloudQuery container. * These are executed first. * - * The containers filesystem is mostly read-only. If you need to write files you can use the /data folder. + * The containers filesystem is mostly read-only. If you need to write files you can use the /usr/share/cloudquery folder. */ additionalCommands?: string[]; @@ -191,8 +194,6 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { }, }); - const volumePath = '/data'; - const cloudqueryTask = task.addContainer(`${id}Container`, { image: Images.cloudquery, entryPoint: [''], @@ -218,9 +219,9 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { '-c', [ ...additionalCommands, - `printf '${dump(sourceConfig)}' > ${volumePath}/source.yaml`, - `printf '${dump(destinationConfig)}' > ${volumePath}/destination.yaml`, - `/app/cloudquery sync ${volumePath}/source.yaml ${volumePath}/destination.yaml --log-format json --log-console --no-log-file`, + `printf '${dump(sourceConfig)}' > ${serviceCatalogueConfigDirectory}/source.yaml`, + `printf '${dump(destinationConfig)}' > ${serviceCatalogueConfigDirectory}/destination.yaml`, + `/app/cloudquery sync ${serviceCatalogueConfigDirectory}/source.yaml ${serviceCatalogueConfigDirectory}/destination.yaml --log-format json --log-console --no-log-file`, ].join(';'), ], logging: fireLensLogDriver, @@ -239,7 +240,7 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { cloudqueryTask.addMountPoints( { // So that we can write task config to this directory - containerPath: volumePath, + containerPath: serviceCatalogueConfigDirectory, sourceVolume: 'config-volume', readOnly: false, }, From 9aea56e1bb0f19945b914d85c335a86209fe1524 Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:46:24 +0100 Subject: [PATCH 19/21] update old /data paths --- .../service-catalogue.test.ts.snap | 18 +++++++++--------- packages/cdk/lib/cloudquery/config.test.ts | 6 +++--- packages/cdk/lib/cloudquery/config.ts | 12 +++++++++--- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 7f010f5df..adab69bfc 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -3665,9 +3665,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /data/github-private-key - app_id: \${file:/data/github-app-id} - installation_id: \${file:/data/github-installation-id} + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -4770,9 +4770,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /data/github-private-key - app_id: \${file:/data/github-app-id} - installation_id: \${file:/data/github-installation-id} + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -5470,9 +5470,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /data/github-private-key - app_id: \${file:/data/github-app-id} - installation_id: \${file:/data/github-installation-id} + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql diff --git a/packages/cdk/lib/cloudquery/config.test.ts b/packages/cdk/lib/cloudquery/config.test.ts index a134a19d7..c8bd1b289 100644 --- a/packages/cdk/lib/cloudquery/config.test.ts +++ b/packages/cdk/lib/cloudquery/config.test.ts @@ -196,9 +196,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /data/github-private-key - app_id: \${file:/data/github-app-id} - installation_id: \${file:/data/github-installation-id} + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} " `); }); diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index 159bf1ebf..62c2f28b5 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -143,9 +143,15 @@ export function githubSourceConfig( // For simplicity, read all configuration from disk. //These values are duplicated. Can we use a variable for them? - private_key_path: '/data/github-private-key', - app_id: '${file:/data/github-app-id}', - installation_id: '${file:/data/github-installation-id}', + private_key_path: `${serviceCatalogueConfigDirectory}/github-private-key`, + app_id: + '${' + + `file:${serviceCatalogueConfigDirectory}/github-app-id` + + '}', + installation_id: + '${' + + `file:${serviceCatalogueConfigDirectory}/github-installation-id` + + '}', }, ], }, From b572c37e6084eb44095ae6adca1e2fef6e836532 Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:51:35 +0100 Subject: [PATCH 20/21] remove debug comment --- packages/cdk/lib/cloudquery/config.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index 62c2f28b5..6d541ed36 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -142,7 +142,6 @@ export function githubSourceConfig( org: 'guardian', // For simplicity, read all configuration from disk. - //These values are duplicated. Can we use a variable for them? private_key_path: `${serviceCatalogueConfigDirectory}/github-private-key`, app_id: '${' + From fed3406aee9c8e63ca0565cc27bca3a1ee58f2a2 Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:54:53 +0100 Subject: [PATCH 21/21] use commit hashes instead of versions --- .github/workflows/cq-image.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cq-image.yml b/.github/workflows/cq-image.yml index 9d3f07d77..050d31a50 100644 --- a/.github/workflows/cq-image.yml +++ b/.github/workflows/cq-image.yml @@ -34,13 +34,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -48,7 +48,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | @@ -60,7 +60,7 @@ jobs: echo "CQ_CLI=${CQ_CLI}" >> "$GITHUB_ENV" - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 with: context: ./ file: containers/cloudquery/Dockerfile