-
Notifications
You must be signed in to change notification settings - Fork 64
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #4762 from consideRatio/pr/nmfs-openscapes
nmfs-openscapes: add cluster files
- Loading branch information
Showing
11 changed files
with
388 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| name: nmfs-openscapes | ||
| provider: aws # https://891612562472.signin.aws.amazon.com/console | ||
| aws: | ||
| key: enc-deployer-credentials.secret.json | ||
| clusterType: eks | ||
| clusterName: nmfs-openscapes | ||
| region: us-west-2 | ||
| billing: | ||
| paid_by_us: false | ||
| support: | ||
| helm_chart_values_files: | ||
| - support.values.yaml | ||
| - enc-support.secret.values.yaml | ||
| hubs: | ||
| [] | ||
| # Uncomment the lines below once the support infrastructure was deployed and | ||
| # you are ready to add the first cluster | ||
|
|
||
| # - name: <hub_name> | ||
| # # Tip: consider changing this to something more human friendly | ||
| # display_name: "nmfs-openscapes - <hub_name>" | ||
| # domain: <hub_name>.nmfs-openscapes.2i2c.cloud | ||
| # helm_chart: basehub | ||
| # helm_chart_values_files: | ||
| # - common.values.yaml | ||
| # - <hub_name>.values.yaml | ||
| # - enc-<hub_name>.secret.values.yaml |
25 changes: 25 additions & 0 deletions
25
config/clusters/nmfs-openscapes/enc-deployer-credentials.secret.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| { | ||
| "AccessKey": { | ||
| "AccessKeyId": "ENC[AES256_GCM,data:97VM/eSCAMnEWvcj2pOBTVdCI/M=,iv:yaQfp5fGzJgbbkbSqbGyUnn49Y0zBdUf6qj2MQUe5e4=,tag:tGqMBMbsL7VMT8PKN3e2fA==,type:str]", | ||
| "SecretAccessKey": "ENC[AES256_GCM,data:IzxyWJNvqI4lFzuPbC/b+3w2eSWAuDk58cyi7bU0EPxQ1TefK/LTtg==,iv:z28cJYV/066dtpInIJ2fpwO/8ti8o98YpAZVEnFYlPQ=,tag:Df/tLa6dwXvvf3XWDndtuQ==,type:str]", | ||
| "UserName": "ENC[AES256_GCM,data:afMU/OOXogtbb3XDrtUb5j0Ij3qydWI=,iv:l1Dh459VjD53M1hnhqyhY16w2c4J+FgOpjeuENqQkGc=,tag:ClWtdIEfTjAolwiQZD14CQ==,type:str]" | ||
| }, | ||
| "sops": { | ||
| "kms": null, | ||
| "gcp_kms": [ | ||
| { | ||
| "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", | ||
| "created_at": "2024-09-09T14:44:20Z", | ||
| "enc": "CiUA4OM7eNCK+1vrOfGfoiorEq68Zyw0ttYDYOcV1aO5r5RFEmBDEkkA5dG1Q09E32GJL/EniI5GgKcdjY4WmD02cPHKkYWF/kyvc7OIGnyYqDm53nLl/ITHs/gi0TroovJ9xpgKTomZfQwY5qA8pv6S" | ||
| } | ||
| ], | ||
| "azure_kv": null, | ||
| "hc_vault": null, | ||
| "age": null, | ||
| "lastmodified": "2024-09-09T14:44:21Z", | ||
| "mac": "ENC[AES256_GCM,data:+vDQFVgWJV5dGUw/tyUdRL55w/Amx7wXCPxK7em7iI2JQqWdPZ0Qv8qhz5RyIXVLlzgQuFkzRR2jvXZ3G8kIT0MQsmx5Vcnf6+MEQrxroFnG1avvQkBXEzIHkMYgDoBAM+zb+mauq3mFNGg3antiVJ9shmlBlnMBkwoDSewBWz8=,iv:vgjDRoAMVu59Rq+gFQNK2Ty5mM7XQ2wGf6PysPqqI9M=,tag:di/pmrQxkhxP5IGNznUtLw==,type:str]", | ||
| "pgp": null, | ||
| "unencrypted_suffix": "_unencrypted", | ||
| "version": "3.9.0" | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
config/clusters/nmfs-openscapes/enc-grafana-token.secret.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| grafana_token: ENC[AES256_GCM,data:oltr88zTHpqrsAVZbgXMKaCuXlu5WXL3vKqTBvdWnkHPRu19V464JxbIxv1y7Q==,iv:OrV7XtJv9rxSggzdbOkfFGuNAEjzL8DMsm0idjiItIE=,tag:rUfqZ/wjFZXCqyTgm4fT/A==,type:str] | ||
| sops: | ||
| kms: [] | ||
| gcp_kms: | ||
| - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs | ||
| created_at: "2024-09-09T22:40:15Z" | ||
| enc: CiUA4OM7eNkMFbgu/SwLgFDVjyRsMDeExbYpvSPxcSrtqqKLbY+jEkkA5dG1QwO/eaZ+abH604bDUh345L8ylLNA2NYGv91osF+nES1yLOvFRcJnfb4XewcZCqfRM3rqLkL+J/ysFP0kiQbnkCh6LWdx | ||
| azure_kv: [] | ||
| hc_vault: [] | ||
| age: [] | ||
| lastmodified: "2024-09-09T22:40:16Z" | ||
| mac: ENC[AES256_GCM,data:roFEK54gMLAE6wTvNpVkzKFOcKni3Ve0zvE23KoCeK8/mFDko36FSEYv3dH8Rt00fafz/MB2B+P5VUqslL3ZKlVJVqYeGThXOzHnEx+ncMgkSCI+hYGRM9yg4vEM7QbKQnSTKm2ALuA084R8FOQZ8fzxF4Jqv7WPne4df13PZgc=,iv:JIVOhyk0RHaxFSrEHUTiU1rr98UyDouptuLotTTZNrQ=,tag:pNwdsw6z3ZGrWL9wMalapw==,type:str] | ||
| pgp: [] | ||
| unencrypted_suffix: _unencrypted | ||
| version: 3.9.0 |
22 changes: 22 additions & 0 deletions
22
config/clusters/nmfs-openscapes/enc-support.secret.values.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| prometheusIngressAuthSecret: | ||
| username: ENC[AES256_GCM,data:F3KK/u3+bOB/OAFOPYroZmHVc0qGlibCCe1noQGl80Lkx6QniYKUmb1/+qC4bwss9XCcUN4svg17r1Lhwk+JaA==,iv:8ZVb7VLPbPd01gqtbNJisuuKHCN1NwaDHAuou0hQViw=,tag:QHlDvrqJrYO/MGKEvBFHmw==,type:str] | ||
| password: ENC[AES256_GCM,data:Y+piow5Z5Q+H0xKlxuAGPY/+FnBObKEhp+OSEtZzsT0sxIm6G85N5xMCzNfJPxCV6PN1zqBlOCG1j7BCC0sgxA==,iv:AIu8gJnuDEDJ9AGn94BpTmVEkTC2gHWB1J2hPO8JVe0=,tag:lOubehYppiU1GvRI4NgR/g==,type:str] | ||
| grafana: | ||
| grafana.ini: | ||
| auth.github: | ||
| client_id: ENC[AES256_GCM,data:ZIHGmzs39bFwjW++wAJLIta+erk=,iv:sfwCcqskbOH8loKI2vVrgWvVYXOkjcsIgv/U3bZZ46M=,tag:A+ejPIdgMtVHL21j3b+nZg==,type:str] | ||
| client_secret: ENC[AES256_GCM,data:C2yVrbA6ob1witb6pN5iPwaUQRg2rbphFHZqaOVVPWk9MxvRwe2BWg==,iv:d2xeZbAYFtyaW/llglUUkm7Xoid2C+UlQO8L8kiUWVM=,tag:AC44iost6zQHN+Z1408hgw==,type:str] | ||
| sops: | ||
| kms: [] | ||
| gcp_kms: | ||
| - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs | ||
| created_at: "2024-09-09T10:19:58Z" | ||
| enc: CiUA4OM7eCz+iFQj+uFqEm/mDksULTmCzUeclG0q1ROLfaU+6xNQEkkA5dG1Q4kplLxCZ8sX6NcGOLqceM4cobPg0RCXK9SL6nvzvRNigf79dIeRZMnRa92K5k5d5CPW6BgheMHqx1LEd7PuvS3mOy3G | ||
| azure_kv: [] | ||
| hc_vault: [] | ||
| age: [] | ||
| lastmodified: "2024-09-09T19:14:50Z" | ||
| mac: ENC[AES256_GCM,data:4amQeYczynHRsyLEmNJaouTXkmi2/vfDZuNeA2XpeJz7y61d6VWnk7CXzWukxdFYStKuFCg0RSpxNqel4NJRfEd9f5SwuQr0l5wYVQ1q31h8G8ZXUjckeZRC8jdYUozJzvAaibkdGUZ8ldS4uPJPteLUoL+osYn0T6BxZDp8Zkg=,iv:61jkt0Zdp3GAFij3qYQ7bcnQbdfAk0Fzu/YP7/bHMF4=,tag:ir0oGyBL5FYXNe/KYBGSTg==,type:str] | ||
| pgp: [] | ||
| unencrypted_suffix: _unencrypted | ||
| version: 3.9.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| prometheusIngressAuthSecret: | ||
| enabled: true | ||
|
|
||
| prometheus: | ||
| server: | ||
| ingress: | ||
| enabled: true | ||
| hosts: | ||
| - prometheus.nmfs-openscapes.2i2c.cloud | ||
| tls: | ||
| - secretName: prometheus-tls | ||
| hosts: | ||
| - prometheus.nmfs-openscapes.2i2c.cloud | ||
| resources: | ||
| requests: | ||
| memory: 8Gi | ||
| limits: | ||
| memory: 8Gi | ||
|
|
||
| grafana: | ||
| grafana.ini: | ||
| server: | ||
| root_url: https://grafana.nmfs-openscapes.2i2c.cloud/ | ||
| auth.github: | ||
| enabled: true | ||
| allowed_organizations: 2i2c-org nmfs-openscapes | ||
| ingress: | ||
| hosts: | ||
| - grafana.nmfs-openscapes.2i2c.cloud | ||
| tls: | ||
| - secretName: grafana-tls | ||
| hosts: | ||
| - grafana.nmfs-openscapes.2i2c.cloud | ||
|
|
||
| cluster-autoscaler: | ||
| enabled: true | ||
| autoDiscovery: | ||
| clusterName: nmfs-openscapes | ||
| awsRegion: us-west-2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| /* | ||
| This file is a jsonnet template of a eksctl's cluster configuration file, | ||
| that is used with the eksctl CLI to both update and initialize an AWS EKS | ||
| based cluster. | ||
| This file has in turn been generated from eksctl/template.jsonnet which is | ||
| relevant to compare with for changes over time. | ||
| To use jsonnet to generate an eksctl configuration file from this, do: | ||
| jsonnet nmfs-openscapes.jsonnet > nmfs-openscapes.eksctl.yaml | ||
| References: | ||
| - https://eksctl.io/usage/schema/ | ||
| */ | ||
| local ng = import "./libsonnet/nodegroup.jsonnet"; | ||
|
|
||
| // place all cluster nodes here | ||
| local clusterRegion = "us-west-2"; | ||
| local masterAzs = ["us-west-2a", "us-west-2b", "us-west-2c"]; | ||
| local nodeAz = "us-west-2b"; | ||
|
|
||
| // Node definitions for notebook nodes. Config here is merged | ||
| // with our notebook node definition. | ||
| // A `node.kubernetes.io/instance-type label is added, so pods | ||
| // can request a particular kind of node with a nodeSelector | ||
| local notebookNodes = [ | ||
| { | ||
| instanceType: "r7i.xlarge", | ||
| namePrefix: "nb-staging", | ||
| labels+: { "2i2c.org/hub-name": "staging" }, | ||
| tags+: { "2i2c:hub-name": "staging" }, | ||
| }, | ||
| { | ||
| instanceType: "r7i.4xlarge", | ||
| namePrefix: "nb-staging", | ||
| labels+: { "2i2c.org/hub-name": "staging" }, | ||
| tags+: { "2i2c:hub-name": "staging" }, | ||
| }, | ||
| { | ||
| instanceType: "r7i.16xlarge", | ||
| namePrefix: "nb-staging", | ||
| labels+: { "2i2c.org/hub-name": "staging" }, | ||
| tags+: { "2i2c:hub-name": "staging" }, | ||
| }, | ||
| { | ||
| instanceType: "r7i.xlarge", | ||
| namePrefix: "nb-prod", | ||
| labels+: { "2i2c.org/hub-name": "prod" }, | ||
| tags+: { "2i2c:hub-name": "prod" }, | ||
| }, | ||
| { | ||
| instanceType: "r7i.4xlarge", | ||
| namePrefix: "nb-prod", | ||
| labels+: { "2i2c.org/hub-name": "prod" }, | ||
| tags+: { "2i2c:hub-name": "prod" }, | ||
| }, | ||
| { | ||
| instanceType: "r7i.16xlarge", | ||
| namePrefix: "nb-prod", | ||
| labels+: { "2i2c.org/hub-name": "prod" }, | ||
| tags+: { "2i2c:hub-name": "prod" }, | ||
| }, | ||
| ]; | ||
| local daskNodes = []; | ||
|
|
||
|
|
||
| { | ||
| apiVersion: 'eksctl.io/v1alpha5', | ||
| kind: 'ClusterConfig', | ||
| metadata+: { | ||
| name: "nmfs-openscapes", | ||
| region: clusterRegion, | ||
| version: "1.30", | ||
| tags+: { | ||
| "ManagedBy": "2i2c", | ||
| "2i2c.org/cluster-name": $.metadata.name, | ||
| }, | ||
| }, | ||
| availabilityZones: masterAzs, | ||
| iam: { | ||
| withOIDC: true, | ||
| }, | ||
| // If you add an addon to this config, run the create addon command. | ||
| // | ||
| // eksctl create addon --config-file=nmfs-openscapes.eksctl.yaml | ||
| // | ||
| addons: [ | ||
| { version: "latest", tags: $.metadata.tags } + addon | ||
| for addon in | ||
| [ | ||
| { | ||
| name: "vpc-cni", | ||
| # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml | ||
| configurationValues: ||| | ||
| enableNetworkPolicy: "true" | ||
| |||, | ||
| }, | ||
| { name: "coredns" }, | ||
| { name: "kube-proxy" }, | ||
| { | ||
| // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that | ||
| // couple to AWS EBS based storage, without it expect to see pods | ||
| // mounting a PVC failing to schedule and PVC resources that are | ||
| // unbound. | ||
| // | ||
| // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html | ||
| // | ||
| name: "aws-ebs-csi-driver", | ||
| wellKnownPolicies: { | ||
| ebsCSIController: true, | ||
| }, | ||
| # configurationValues ref: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/HEAD/charts/aws-ebs-csi-driver/values.yaml | ||
| configurationValues: ||| | ||
| defaultStorageClass: | ||
| enabled: true | ||
| |||, | ||
| }, | ||
| ] | ||
| ], | ||
| nodeGroups: [ | ||
| n + {clusterName: $.metadata.name} for n in | ||
| [ | ||
| ng + { | ||
| namePrefix: 'core', | ||
| nameSuffix: 'a', | ||
| nameIncludeInstanceType: false, | ||
| availabilityZones: [nodeAz], | ||
| ssh: { | ||
| publicKeyPath: 'ssh-keys/nmfs-openscapes.key.pub' | ||
| }, | ||
| instanceType: "r7i.xlarge", | ||
| minSize: 1, | ||
| maxSize: 6, | ||
| labels+: { | ||
| "hub.jupyter.org/node-purpose": "core", | ||
| "k8s.dask.org/node-purpose": "core", | ||
| }, | ||
| }, | ||
| ] + [ | ||
| ng + { | ||
| namePrefix: 'nb', | ||
| availabilityZones: [nodeAz], | ||
| minSize: 0, | ||
| maxSize: 500, | ||
| instanceType: n.instanceType, | ||
| ssh: { | ||
| publicKeyPath: 'ssh-keys/nmfs-openscapes.key.pub' | ||
| }, | ||
| labels+: { | ||
| "hub.jupyter.org/node-purpose": "user", | ||
| "k8s.dask.org/node-purpose": "scheduler" | ||
| }, | ||
| taints+: { | ||
| "hub.jupyter.org_dedicated": "user:NoSchedule", | ||
| "hub.jupyter.org/dedicated": "user:NoSchedule", | ||
| }, | ||
| } + n for n in notebookNodes | ||
| ] + ( if daskNodes != null then | ||
| [ | ||
| ng + { | ||
| namePrefix: 'dask', | ||
| availabilityZones: [nodeAz], | ||
| minSize: 0, | ||
| maxSize: 500, | ||
| ssh: { | ||
| publicKeyPath: 'ssh-keys/nmfs-openscapes.key.pub' | ||
| }, | ||
| labels+: { | ||
| "k8s.dask.org/node-purpose": "worker" | ||
| }, | ||
| taints+: { | ||
| "k8s.dask.org_dedicated" : "worker:NoSchedule", | ||
| "k8s.dask.org/dedicated" : "worker:NoSchedule", | ||
| }, | ||
| instancesDistribution+: { | ||
| onDemandBaseCapacity: 0, | ||
| onDemandPercentageAboveBaseCapacity: 0, | ||
| spotAllocationStrategy: "capacity-optimized", | ||
| }, | ||
| } + n for n in daskNodes | ||
| ] else [] | ||
| ) | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB3Jjg/Gf5OZeErLJc5pFuBra3OS7Rkevux8AsdtxpnP erik@dl |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "data": "ENC[AES256_GCM,data:m4Polf8TlCmwJGlI8wwqx66tgTI35ZN0YZ7gT1syYFC7MlUyqoUi8yBdRBLFGGHa6sYdmUqhosHzOe9lxvugvASO51mPB4OhKOSQm+sTGtTPXjcB0G9CkF5N7pBwtZjaQrF6ZK2ybg6BpRSxpy14Q/HZF7hHeMEhXe9rU2PBQl3t9FOl+2T8jEYddk7f+jkFPPxaPKsDSq2q59Uh6zmV+UhzJb60gaD10hhdotyRT6GkENHv2kLGv/Ok+DzV6FkWIpA3BkdkqeqTtulziNvqKsg0bpbfctW2hyE0Sz109P0mKHrDTR293v23RoOt5LL1myxdkjfzyPZlmFG/9OAO1H7OFI9xCgBAAwcq4D51+H9m5GpfAOgQVSVEC7N0+QKQ5AUBH6lSiJRazB7Z7lgcjidNriYQ2bkEo7L9iNi4+X1oxX0F3HyZS+QKdwowWd4y7iet3i9TQcxImEn5oKLQ2zXzWDJMUuUTF+Snz0EcSzXXWv/gaJUkMnMP+7VPMnu72riMyDjjbL2VgQohfhMV,iv:20EEI2rVg7Ai7tKrYfJBoUcAbtPSDFXeCbKL+TLaVIQ=,tag:Bo5kMjbQztL/C7x0hcYqQg==,type:str]", | ||
| "sops": { | ||
| "kms": null, | ||
| "gcp_kms": [ | ||
| { | ||
| "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", | ||
| "created_at": "2024-09-09T10:19:58Z", | ||
| "enc": "CiUA4OM7eJrRZedVQv3ik1WgL5K3J/UJ4LVFD0JoNSz6t6QMUQUVEkkA5dG1Q0KPwwWtI8m+vuDMM6QliIJis11EgP97jWQHYGf0Xj8Hbd17jf8WdZBurBxg+hJI1hOTuGU56xXKSb3AzIzPXRVZ3AZR" | ||
| } | ||
| ], | ||
| "azure_kv": null, | ||
| "hc_vault": null, | ||
| "age": null, | ||
| "lastmodified": "2024-09-09T10:19:58Z", | ||
| "mac": "ENC[AES256_GCM,data:RFSHuuHsDSdWOPWc41dWAVs7rmti32qsCgK+nCMRQBm8DWsPVhRUmxmY16tnzLIqufF9HnyixaaPRTXUB9npywsMqrvkHjVfHamsoodTKTyqor21YyUWj6opbZfolDxoqZfCX1RG/+DUMaizKx9bDOolfSX5ARSlgN6F9LqmcE8=,iv:34gkfjyTzNZ+/QZ5LtsnQosqBYH4ZTYJFCexeG3+Teg=,tag:6QgLxIrZiG5XcOLT3ZE5xA==,type:str]", | ||
| "pgp": null, | ||
| "unencrypted_suffix": "_unencrypted", | ||
| "version": "3.9.0" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.