From 75116e76ee51a3d4107605cfe87ec39ef5d21ac5 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Sun, 14 Apr 2024 19:16:54 +0200 Subject: [PATCH 01/10] initial commit for iac/oci-prow-worker Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/.gitignore | 2 ++ iac/oci-prow-worker/.terraform.lock.hcl | 24 ++++++++++++++++++++++++ iac/oci-prow-worker/Makefile | 4 ++++ iac/oci-prow-worker/README.md | 3 +++ iac/oci-prow-worker/cluster.tf | 17 +++++++++++++++++ iac/oci-prow-worker/provider.tf | 6 ++++++ iac/oci-prow-worker/variables.tf | 20 ++++++++++++++++++++ 7 files changed, 76 insertions(+) create mode 100644 iac/oci-prow-worker/.gitignore create mode 100644 iac/oci-prow-worker/.terraform.lock.hcl create mode 100644 iac/oci-prow-worker/Makefile create mode 100644 iac/oci-prow-worker/README.md create mode 100644 iac/oci-prow-worker/cluster.tf create mode 100644 iac/oci-prow-worker/provider.tf create mode 100644 iac/oci-prow-worker/variables.tf diff --git a/iac/oci-prow-worker/.gitignore b/iac/oci-prow-worker/.gitignore new file mode 100644 index 0000000..1ed4733 --- /dev/null +++ b/iac/oci-prow-worker/.gitignore @@ -0,0 +1,2 @@ +.terraform +*.tfvars diff --git a/iac/oci-prow-worker/.terraform.lock.hcl b/iac/oci-prow-worker/.terraform.lock.hcl new file mode 100644 index 0000000..a4a13bc --- /dev/null +++ b/iac/oci-prow-worker/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/oci" { + version = "5.36.0" + hashes = [ + "h1:UAh0wGPAa8p/A8YQ/UUcFpkwdtj7AGE/WZyqQfQqwig=", + "zh:1fe8a3fc210bae48658c703dd8aa458f794aab983dca1d591f9158e12e2dd5a2", + "zh:2d2bc52560cd87403f4ab287c0cc1577e3735028d1028a54830113b8537c36f4", + "zh:4783b0db1ad0882abf4637e30db3cfbd69a23d72355fe1fe5c580606b9c67ea5", + "zh:48e07c4a8c085b68f5cdaaeef218578dc3e4ede068542e0aef16a5eaa6a37cd5", + "zh:61a4cb9a0d7f0e02abe5049cc0a47167371b1391a0b94e5f21a99b80cd0a9bcc", + "zh:6a2206590a8aad7b091a496f80aee84e1da682ead2f3e98e79f895d0dc75e328", + "zh:83bb26f43377ec0bc12d74046e857d40696567defb43927e30a108c81126d4a9", + "zh:914d03e361a49fd296bafa7e10b0c228a5fb5e4f374078670f656166e8026700", + "zh:9749c9638c520e341726f981884d70f81025e368cb150a9b7cde7dc3f1f9c22b", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9ceb0432160c11143e2556170f11c093ebbb088c2161a99eea105f6cb0c7e26a", + "zh:b7289a754153995187c887012f35c010bb8b23aed14bd5806c43ecc51602e266", + "zh:c5e81ed93f94361d8edc528250353f51e842e16ea1731d98919349b7bb30bd27", + "zh:e38b7a6d0b10fd01d6234c7e2c3f7595df791ea96c1f57ee24294f8758ee8fa6", + "zh:e3b6dbf42223d9f87f12345f74996932c56ae941fa4186ae7f7a1f3695284b4f", + ] +} diff --git a/iac/oci-prow-worker/Makefile b/iac/oci-prow-worker/Makefile new file mode 100644 index 0000000..e5b865b --- /dev/null +++ b/iac/oci-prow-worker/Makefile @@ -0,0 +1,4 @@ +OPENTOFU_CLI ?= tofu + +fmt: + $(OPENTOFU_CLI) fmt diff --git a/iac/oci-prow-worker/README.md b/iac/oci-prow-worker/README.md new file mode 100644 index 0000000..419f631 --- /dev/null +++ b/iac/oci-prow-worker/README.md @@ -0,0 +1,3 @@ +# oci-prow-cluster + +This directory deploys the `oci-prow-cluster` OKE cluster in OCI (Oracle Cloud) via [OpenTofu](https://opentofu.org) diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf new file mode 100644 index 0000000..e39fe16 --- /dev/null +++ b/iac/oci-prow-worker/cluster.tf @@ -0,0 +1,17 @@ +resource "oci_containerengine_cluster" "prow" { + name = "oci-prow-worker" + type = "TODO" + kubernetes_version = "v1.29.1" + + compartment_id = var.oci_compartment_id + vcn_id = "TODO" + + cluster_pod_network_options { + cni_type = "flannel" + } + endpoint_config { + is_public_ip_enabled = true + nsg_ids = "TODO" + subnet_id = "TODO" + } +} diff --git a/iac/oci-prow-worker/provider.tf b/iac/oci-prow-worker/provider.tf new file mode 100644 index 0000000..31c9787 --- /dev/null +++ b/iac/oci-prow-worker/provider.tf @@ -0,0 +1,6 @@ +provider "oci" { + tenancy_ocid = "" + user_ocid = "" + private_key = "" + region = "" +} diff --git a/iac/oci-prow-worker/variables.tf b/iac/oci-prow-worker/variables.tf new file mode 100644 index 0000000..3adc762 --- /dev/null +++ b/iac/oci-prow-worker/variables.tf @@ -0,0 +1,20 @@ +variable "oci_tenant_ocid" { + type = string +} + +variable "oci_compartment_id" { + type = string +} + +variable "oci_user_ocid" { + type = string +} + +variable "oci_private_key" { + type = string + sensitive = true +} + +variable "oci_region" { + type = string +} From 2401e5e3adde9ee0ab689ab082029b8562323b4e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Sun, 14 Apr 2024 19:33:52 +0200 Subject: [PATCH 02/10] Use oracle/oci provider Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/.terraform.lock.hcl | 7 ++++--- iac/oci-prow-worker/terraform.tf | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 iac/oci-prow-worker/terraform.tf diff --git a/iac/oci-prow-worker/.terraform.lock.hcl b/iac/oci-prow-worker/.terraform.lock.hcl index a4a13bc..e6dad50 100644 --- a/iac/oci-prow-worker/.terraform.lock.hcl +++ b/iac/oci-prow-worker/.terraform.lock.hcl @@ -1,8 +1,9 @@ -# This file is maintained automatically by "tofu init". +# This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.opentofu.org/hashicorp/oci" { - version = "5.36.0" +provider "registry.terraform.io/oracle/oci" { + version = "5.36.0" + constraints = ">= 4.119.0" hashes = [ "h1:UAh0wGPAa8p/A8YQ/UUcFpkwdtj7AGE/WZyqQfQqwig=", "zh:1fe8a3fc210bae48658c703dd8aa458f794aab983dca1d591f9158e12e2dd5a2", diff --git a/iac/oci-prow-worker/terraform.tf b/iac/oci-prow-worker/terraform.tf new file mode 100644 index 0000000..17461e3 --- /dev/null +++ b/iac/oci-prow-worker/terraform.tf @@ -0,0 +1,9 @@ +terraform { + + required_providers { + oci = { + source = "oracle/oci" + version = "5.36.0" + } + } +} From 3b7a5cffbab1722d7b785fecd10dc2b36ffe772e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Sun, 14 Apr 2024 19:51:39 +0200 Subject: [PATCH 03/10] Add some networking primitives Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/.terraform.lock.hcl | 6 ++--- iac/oci-prow-worker/cluster.tf | 2 +- iac/oci-prow-worker/network.tf | 36 +++++++++++++++++++++++++ iac/oci-prow-worker/provider.tf | 5 ++++ iac/oci-prow-worker/terraform.tf | 4 +-- iac/oci-prow-worker/variables.tf | 2 +- 6 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 iac/oci-prow-worker/network.tf diff --git a/iac/oci-prow-worker/.terraform.lock.hcl b/iac/oci-prow-worker/.terraform.lock.hcl index e6dad50..69f69d4 100644 --- a/iac/oci-prow-worker/.terraform.lock.hcl +++ b/iac/oci-prow-worker/.terraform.lock.hcl @@ -1,9 +1,9 @@ -# This file is maintained automatically by "terraform init". +# This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/oracle/oci" { +provider "registry.opentofu.org/oracle/oci" { version = "5.36.0" - constraints = ">= 4.119.0" + constraints = "5.36.0" hashes = [ "h1:UAh0wGPAa8p/A8YQ/UUcFpkwdtj7AGE/WZyqQfQqwig=", "zh:1fe8a3fc210bae48658c703dd8aa458f794aab983dca1d591f9158e12e2dd5a2", diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf index e39fe16..fa49b8f 100644 --- a/iac/oci-prow-worker/cluster.tf +++ b/iac/oci-prow-worker/cluster.tf @@ -3,7 +3,7 @@ resource "oci_containerengine_cluster" "prow" { type = "TODO" kubernetes_version = "v1.29.1" - compartment_id = var.oci_compartment_id + compartment_id = var.oci_compartment_ocid vcn_id = "TODO" cluster_pod_network_options { diff --git a/iac/oci-prow-worker/network.tf b/iac/oci-prow-worker/network.tf new file mode 100644 index 0000000..540d237 --- /dev/null +++ b/iac/oci-prow-worker/network.tf @@ -0,0 +1,36 @@ +resource "oci_core_vcn" "prow" { + cidr_block = "10.0.0.0/16" + compartment_id = var.oci_compartment_ocid + display_name = "Prow Network" +} + +resource "oci_core_internet_gateway" "prow" { + compartment_id = var.oci_compartment_ocid + display_name = "Prow Internet Gateway" + vcn_id = oci_core_vcn.prow.id +} + +resource "oci_core_route_table" "prow_worker" { + compartment_id = var.oci_compartment_ocid + vcn_id = oci_core_vcn.prow.id + display_name = "Prow Worker Route Table" + + route_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_internet_gateway.prow.id + } +} + +resource "oci_core_subnet" "prow_worker_cluster" { + count = length(data.oci_identity_availability_domains.availability_domains) + + availability_domain = data.oci_identity_availability_domains.availability_domains[count.index].name + cidr_block = "10.0.${20 + count.index}.0/24" + compartment_id = var.oci_compartment_ocid + vcn_id = oci_core_vcn.prow.id + + security_list_ids = [oci_core_vcn.prow.default_security_list_id] + display_name = "Prow Cluster Subnet" + route_table_id = oci_core_route_table.prow_worker.id +} diff --git a/iac/oci-prow-worker/provider.tf b/iac/oci-prow-worker/provider.tf index 31c9787..f3cafb6 100644 --- a/iac/oci-prow-worker/provider.tf +++ b/iac/oci-prow-worker/provider.tf @@ -4,3 +4,8 @@ provider "oci" { private_key = "" region = "" } + +data "oci_identity_availability_domains" "availability_domains" { + compartment_id = var.oci_tenant_ocid +} + diff --git a/iac/oci-prow-worker/terraform.tf b/iac/oci-prow-worker/terraform.tf index 17461e3..4555f8a 100644 --- a/iac/oci-prow-worker/terraform.tf +++ b/iac/oci-prow-worker/terraform.tf @@ -2,8 +2,8 @@ terraform { required_providers { oci = { - source = "oracle/oci" - version = "5.36.0" + source = "oracle/oci" + version = "5.36.0" } } } diff --git a/iac/oci-prow-worker/variables.tf b/iac/oci-prow-worker/variables.tf index 3adc762..706ed38 100644 --- a/iac/oci-prow-worker/variables.tf +++ b/iac/oci-prow-worker/variables.tf @@ -2,7 +2,7 @@ variable "oci_tenant_ocid" { type = string } -variable "oci_compartment_id" { +variable "oci_compartment_ocid" { type = string } From d90df40ef04b798ed0a8d3a26461b85b7ba0796f Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 7 May 2024 13:02:38 +0200 Subject: [PATCH 04/10] Produce working tofu plan output Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/cluster.tf | 28 ++++++++++++++++++++++------ iac/oci-prow-worker/network.tf | 6 +++--- iac/oci-prow-worker/provider.tf | 6 ++---- iac/oci-prow-worker/terraform.tf | 1 - iac/oci-prow-worker/variables.tf | 6 ++++++ 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf index fa49b8f..13160e5 100644 --- a/iac/oci-prow-worker/cluster.tf +++ b/iac/oci-prow-worker/cluster.tf @@ -1,17 +1,33 @@ resource "oci_containerengine_cluster" "prow" { name = "oci-prow-worker" - type = "TODO" kubernetes_version = "v1.29.1" compartment_id = var.oci_compartment_ocid - vcn_id = "TODO" + vcn_id = oci_core_vcn.prow.id cluster_pod_network_options { cni_type = "flannel" } - endpoint_config { - is_public_ip_enabled = true - nsg_ids = "TODO" - subnet_id = "TODO" +} + +resource "oci_containerengine_node_pool" "prow_worker" { + cluster_id = oci_containerengine_cluster.prow.id + compartment_id = var.oci_compartment_ocid + kubernetes_version = "v1.29.1" + name = "prow-worker" + node_shape = "VM.Standard2.1" + subnet_ids = oci_core_subnet.prow_worker_cluster[*].id + + ssh_public_key = var.node_pool_ssh_public_key + + node_config_details { + size = 3 + dynamic "placement_configs" { + for_each = oci_core_subnet.prow_worker_cluster + content { + availability_domain = data.oci_identity_availability_domains.availability_domains.availability_domains[index(oci_core_subnet.prow_worker_cluster, placement_configs.value)].id + subnet_id = placement_configs.value.id + } + } } } diff --git a/iac/oci-prow-worker/network.tf b/iac/oci-prow-worker/network.tf index 540d237..5a11175 100644 --- a/iac/oci-prow-worker/network.tf +++ b/iac/oci-prow-worker/network.tf @@ -23,14 +23,14 @@ resource "oci_core_route_table" "prow_worker" { } resource "oci_core_subnet" "prow_worker_cluster" { - count = length(data.oci_identity_availability_domains.availability_domains) + count = length(data.oci_identity_availability_domains.availability_domains.availability_domains) - availability_domain = data.oci_identity_availability_domains.availability_domains[count.index].name + availability_domain = data.oci_identity_availability_domains.availability_domains.availability_domains[count.index].name cidr_block = "10.0.${20 + count.index}.0/24" compartment_id = var.oci_compartment_ocid vcn_id = oci_core_vcn.prow.id security_list_ids = [oci_core_vcn.prow.default_security_list_id] - display_name = "Prow Cluster Subnet" route_table_id = oci_core_route_table.prow_worker.id + display_name = "Prow Cluster Subnet ${count.index}" } diff --git a/iac/oci-prow-worker/provider.tf b/iac/oci-prow-worker/provider.tf index f3cafb6..ab52715 100644 --- a/iac/oci-prow-worker/provider.tf +++ b/iac/oci-prow-worker/provider.tf @@ -1,8 +1,6 @@ provider "oci" { - tenancy_ocid = "" - user_ocid = "" - private_key = "" - region = "" + tenancy_ocid = var.oci_tenant_ocid + region = var.oci_region } data "oci_identity_availability_domains" "availability_domains" { diff --git a/iac/oci-prow-worker/terraform.tf b/iac/oci-prow-worker/terraform.tf index 4555f8a..7de03d7 100644 --- a/iac/oci-prow-worker/terraform.tf +++ b/iac/oci-prow-worker/terraform.tf @@ -1,5 +1,4 @@ terraform { - required_providers { oci = { source = "oracle/oci" diff --git a/iac/oci-prow-worker/variables.tf b/iac/oci-prow-worker/variables.tf index 706ed38..9e58363 100644 --- a/iac/oci-prow-worker/variables.tf +++ b/iac/oci-prow-worker/variables.tf @@ -6,6 +6,7 @@ variable "oci_compartment_ocid" { type = string } +/* variable "oci_user_ocid" { type = string } @@ -14,7 +15,12 @@ variable "oci_private_key" { type = string sensitive = true } +*/ variable "oci_region" { type = string } + +variable "node_pool_ssh_public_key" { + type = string +} From 12dd4f872500406b25490a619cbed16ca86c953d Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 7 May 2024 15:57:39 +0200 Subject: [PATCH 05/10] Add state backend in OCI bucket Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/.gitignore | 4 ++++ iac/oci-prow-worker/Makefile | 9 +++++++++ iac/oci-prow-worker/README.md | 10 +++++++++- iac/oci-prow-worker/cluster.tf | 26 +++++++++++++++++--------- iac/oci-prow-worker/terraform.tf | 13 +++++++++++++ iac/oci-prow-worker/variables.tf | 10 ++++++++++ 6 files changed, 62 insertions(+), 10 deletions(-) diff --git a/iac/oci-prow-worker/.gitignore b/iac/oci-prow-worker/.gitignore index 1ed4733..cc49b2f 100644 --- a/iac/oci-prow-worker/.gitignore +++ b/iac/oci-prow-worker/.gitignore @@ -1,2 +1,6 @@ +# Terraform folder .terraform +# Make sure to not allow checking in tfvars by mistake *.tfvars +# Environment variables are often stored in this file +.env diff --git a/iac/oci-prow-worker/Makefile b/iac/oci-prow-worker/Makefile index e5b865b..5b19532 100644 --- a/iac/oci-prow-worker/Makefile +++ b/iac/oci-prow-worker/Makefile @@ -1,4 +1,13 @@ OPENTOFU_CLI ?= tofu +init: + $(OPENTOFU_CLI) init + fmt: $(OPENTOFU_CLI) fmt + +plan: + $(OPENTOFU_CLI) plan + +apply: + $(OPENTOFU_CLI) apply diff --git a/iac/oci-prow-worker/README.md b/iac/oci-prow-worker/README.md index 419f631..fd6c2e9 100644 --- a/iac/oci-prow-worker/README.md +++ b/iac/oci-prow-worker/README.md @@ -1,3 +1,11 @@ # oci-prow-cluster -This directory deploys the `oci-prow-cluster` OKE cluster in OCI (Oracle Cloud) via [OpenTofu](https://opentofu.org) +This directory deploys the `oci-prow-cluster` OKE cluster in OCI (Oracle Cloud) via [OpenTofu](https://opentofu.org). A shared state is stored in a OCI storage bucket, please make sure to use that. Usually, this code shouldn't be executed directly but run by Prow. + +## Required Environment Variables + +The following environment variables are required before running any `make` targets: + +- `AWS_ACCESS_KEY_ID`: Needs to be the key ID for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets. +- `AWS_SECRET_ACCESS_KEY`: Needs to be the secret for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets. +- `AWS_ENDPOINT_URL_S3`: Needs to be `https://.compat.objectstorage.us-sanjose-1.oraclecloud.com`. Replace `` with the namespace displayed on the bucket (see OCI Console for this information). diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf index 13160e5..f0abc97 100644 --- a/iac/oci-prow-worker/cluster.tf +++ b/iac/oci-prow-worker/cluster.tf @@ -1,6 +1,6 @@ resource "oci_containerengine_cluster" "prow" { name = "oci-prow-worker" - kubernetes_version = "v1.29.1" + kubernetes_version = var.kubernetes_version compartment_id = var.oci_compartment_ocid vcn_id = oci_core_vcn.prow.id @@ -11,22 +11,30 @@ resource "oci_containerengine_cluster" "prow" { } resource "oci_containerengine_node_pool" "prow_worker" { - cluster_id = oci_containerengine_cluster.prow.id - compartment_id = var.oci_compartment_ocid - kubernetes_version = "v1.29.1" + cluster_id = oci_containerengine_cluster.prow.id + compartment_id = var.oci_compartment_ocid + subnet_ids = oci_core_subnet.prow_worker_cluster[*].id + + kubernetes_version = var.kubernetes_version name = "prow-worker" - node_shape = "VM.Standard2.1" - subnet_ids = oci_core_subnet.prow_worker_cluster[*].id + ssh_public_key = var.node_pool_ssh_public_key - ssh_public_key = var.node_pool_ssh_public_key + # this matches t3.2xlarge sizings. + node_shape = "VM.Standard.A1.Flex" + node_shape_config { + memory_in_gbs = 32 + ocpus = 8 + } node_config_details { - size = 3 + size = var.node_pool_worker_size + # create placement_configs for each availability domain. + # There happens to be only a single one in us-sanjose-1. dynamic "placement_configs" { for_each = oci_core_subnet.prow_worker_cluster content { availability_domain = data.oci_identity_availability_domains.availability_domains.availability_domains[index(oci_core_subnet.prow_worker_cluster, placement_configs.value)].id - subnet_id = placement_configs.value.id + subnet_id = placement_configs.value.id } } } diff --git a/iac/oci-prow-worker/terraform.tf b/iac/oci-prow-worker/terraform.tf index 7de03d7..3f3bc24 100644 --- a/iac/oci-prow-worker/terraform.tf +++ b/iac/oci-prow-worker/terraform.tf @@ -5,4 +5,17 @@ terraform { version = "5.36.0" } } + + # make sure to set AWS_ENDPOINT_URL_S3 to 'https://.compat.objectstorage.us-sanjose-1.oraclecloud.com'. + backend "s3" { + bucket = "kcp-opentofu-state" + region = "us-sanjose-1" + key = "ci-prow-worker/tf.tfstate" + + skip_region_validation = true + skip_credentials_validation = true + skip_requesting_account_id = true + use_path_style = true + skip_metadata_api_check = true + } } diff --git a/iac/oci-prow-worker/variables.tf b/iac/oci-prow-worker/variables.tf index 9e58363..7da26a3 100644 --- a/iac/oci-prow-worker/variables.tf +++ b/iac/oci-prow-worker/variables.tf @@ -24,3 +24,13 @@ variable "oci_region" { variable "node_pool_ssh_public_key" { type = string } + +variable "node_pool_worker_size" { + type = number + default = 3 +} + +variable "kubernetes_version" { + type = string + default = "v1.29.1" +} From 734698ab46a06d35eb1aee762178ed8316a4a26f Mon Sep 17 00:00:00 2001 From: Mangirdas Judeikis Date: Fri, 7 Jun 2024 21:21:40 +0300 Subject: [PATCH 06/10] nits --- iac/oci-prow-worker/README.md | 26 ++++++++++++++++++++++++++ iac/oci-prow-worker/cluster.tf | 21 +++++++++++++-------- iac/oci-prow-worker/provider.tf | 6 ++++-- iac/oci-prow-worker/variables.tf | 10 ++++++++++ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/iac/oci-prow-worker/README.md b/iac/oci-prow-worker/README.md index fd6c2e9..28c876a 100644 --- a/iac/oci-prow-worker/README.md +++ b/iac/oci-prow-worker/README.md @@ -9,3 +9,29 @@ The following environment variables are required before running any `make` targe - `AWS_ACCESS_KEY_ID`: Needs to be the key ID for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets. - `AWS_SECRET_ACCESS_KEY`: Needs to be the secret for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets. - `AWS_ENDPOINT_URL_S3`: Needs to be `https://.compat.objectstorage.us-sanjose-1.oraclecloud.com`. Replace `` with the namespace displayed on the bucket (see OCI Console for this information). + +## Running terraform + +Easiest way to run terraform locally is to create a `.env` file with the required environment variables and then run `make` commands. For example: + +```bash +export AWS_ACCESS_KEY_ID=xxxxxxxxxxxxxxx +export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxx +export AWS_ENDPOINT_URL_S3=https://xxxxxxxxxxxx.compat.objectstorage.us-sanjose-1.oraclecloud.com +export TF_LOG=DEBUG +``` + +Create `terraform.tfvars` file with the following content: + +```hcl +oci_tenant_ocid = "ocid1.tenancy.oc1..xxxxxxxxxxxxxxxxxxx" +oci_compartment_ocid = "ocid1.compartment.oc1..xxxxxxxxxxxxxxxxxxx" +oci_region = "us-sanjose-1" +node_pool_ssh_public_key = "ssh-rsa " +oci_auth_type = "SecurityToken" +oci_config_file_profile = "KCP" +``` + +Install `oci` cli and run `oci session authenticate` to get the `oci_config_file` and `oci_profile` values. + +Then run `make init` and `make plan` to see the changes that will be applied. If everything looks good, run `make apply`. diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf index f0abc97..939b894 100644 --- a/iac/oci-prow-worker/cluster.tf +++ b/iac/oci-prow-worker/cluster.tf @@ -4,16 +4,11 @@ resource "oci_containerengine_cluster" "prow" { compartment_id = var.oci_compartment_ocid vcn_id = oci_core_vcn.prow.id - - cluster_pod_network_options { - cni_type = "flannel" - } } resource "oci_containerengine_node_pool" "prow_worker" { cluster_id = oci_containerengine_cluster.prow.id compartment_id = var.oci_compartment_ocid - subnet_ids = oci_core_subnet.prow_worker_cluster[*].id kubernetes_version = var.kubernetes_version name = "prow-worker" @@ -26,15 +21,25 @@ resource "oci_containerengine_node_pool" "prow_worker" { ocpus = 8 } + + # Using image Oracle-Linux-7.x- + # Find image OCID for your region from https://docs.oracle.com/iaas/images/ + # For now aarch64 lates k/k 1.29 image is used. + node_source_details { + image_id = "ocid1.image.oc1.us-sanjose-1.aaaaaaaaceb5egr4du2d5vut6uam2kdbctilom4w5wirnz7tihe4w4y3yroq" + source_type = "image" + } + node_config_details { size = var.node_pool_worker_size + # create placement_configs for each availability domain. # There happens to be only a single one in us-sanjose-1. dynamic "placement_configs" { - for_each = oci_core_subnet.prow_worker_cluster + for_each = data.oci_identity_availability_domains.availability_domains.availability_domains content { - availability_domain = data.oci_identity_availability_domains.availability_domains.availability_domains[index(oci_core_subnet.prow_worker_cluster, placement_configs.value)].id - subnet_id = placement_configs.value.id + availability_domain = placement_configs.value.name + subnet_id = oci_core_subnet.prow_worker_cluster[placement_configs.key].id } } } diff --git a/iac/oci-prow-worker/provider.tf b/iac/oci-prow-worker/provider.tf index ab52715..7ceb658 100644 --- a/iac/oci-prow-worker/provider.tf +++ b/iac/oci-prow-worker/provider.tf @@ -1,6 +1,8 @@ provider "oci" { - tenancy_ocid = var.oci_tenant_ocid - region = var.oci_region + tenancy_ocid = var.oci_tenant_ocid + region = var.oci_region + auth = var.oci_auth_type + config_file_profile = var.oci_config_file_profile } data "oci_identity_availability_domains" "availability_domains" { diff --git a/iac/oci-prow-worker/variables.tf b/iac/oci-prow-worker/variables.tf index 7da26a3..3e83dc9 100644 --- a/iac/oci-prow-worker/variables.tf +++ b/iac/oci-prow-worker/variables.tf @@ -34,3 +34,13 @@ variable "kubernetes_version" { type = string default = "v1.29.1" } + +variable "oci_config_file_profile" { + type = string + default = "DEFAULT" +} + +variable "oci_auth_type" { + type = string + default = "SecurityToken" +} From 906f6407a87f2523f740ad57e7d6c0544033bd09 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 16 Jul 2024 10:27:00 +0200 Subject: [PATCH 07/10] Run tofu fmt Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/cluster.tf | 14 +++++++------- iac/oci-prow-worker/variables.tf | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf index 939b894..492d88e 100644 --- a/iac/oci-prow-worker/cluster.tf +++ b/iac/oci-prow-worker/cluster.tf @@ -22,13 +22,13 @@ resource "oci_containerengine_node_pool" "prow_worker" { } - # Using image Oracle-Linux-7.x- - # Find image OCID for your region from https://docs.oracle.com/iaas/images/ - # For now aarch64 lates k/k 1.29 image is used. - node_source_details { - image_id = "ocid1.image.oc1.us-sanjose-1.aaaaaaaaceb5egr4du2d5vut6uam2kdbctilom4w5wirnz7tihe4w4y3yroq" - source_type = "image" - } + # Using image Oracle-Linux-7.x- + # Find image OCID for your region from https://docs.oracle.com/iaas/images/ + # For now aarch64 lates k/k 1.29 image is used. + node_source_details { + image_id = "ocid1.image.oc1.us-sanjose-1.aaaaaaaaceb5egr4du2d5vut6uam2kdbctilom4w5wirnz7tihe4w4y3yroq" + source_type = "image" + } node_config_details { size = var.node_pool_worker_size diff --git a/iac/oci-prow-worker/variables.tf b/iac/oci-prow-worker/variables.tf index 3e83dc9..d22ec6d 100644 --- a/iac/oci-prow-worker/variables.tf +++ b/iac/oci-prow-worker/variables.tf @@ -31,16 +31,16 @@ variable "node_pool_worker_size" { } variable "kubernetes_version" { - type = string + type = string default = "v1.29.1" } variable "oci_config_file_profile" { - type = string + type = string default = "DEFAULT" } variable "oci_auth_type" { - type = string + type = string default = "SecurityToken" } From f77e72b03bff9ee59c21214c2d4f7be91b4faea9 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 16 Jul 2024 16:19:20 +0200 Subject: [PATCH 08/10] Update provider oci to version 6.2.0 Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/.terraform.lock.hcl | 34 ++++++++++++------------- iac/oci-prow-worker/terraform.tf | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/iac/oci-prow-worker/.terraform.lock.hcl b/iac/oci-prow-worker/.terraform.lock.hcl index 69f69d4..a237fab 100644 --- a/iac/oci-prow-worker/.terraform.lock.hcl +++ b/iac/oci-prow-worker/.terraform.lock.hcl @@ -2,24 +2,24 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/oracle/oci" { - version = "5.36.0" - constraints = "5.36.0" + version = "6.2.0" + constraints = "6.2.0" hashes = [ - "h1:UAh0wGPAa8p/A8YQ/UUcFpkwdtj7AGE/WZyqQfQqwig=", - "zh:1fe8a3fc210bae48658c703dd8aa458f794aab983dca1d591f9158e12e2dd5a2", - "zh:2d2bc52560cd87403f4ab287c0cc1577e3735028d1028a54830113b8537c36f4", - "zh:4783b0db1ad0882abf4637e30db3cfbd69a23d72355fe1fe5c580606b9c67ea5", - "zh:48e07c4a8c085b68f5cdaaeef218578dc3e4ede068542e0aef16a5eaa6a37cd5", - "zh:61a4cb9a0d7f0e02abe5049cc0a47167371b1391a0b94e5f21a99b80cd0a9bcc", - "zh:6a2206590a8aad7b091a496f80aee84e1da682ead2f3e98e79f895d0dc75e328", - "zh:83bb26f43377ec0bc12d74046e857d40696567defb43927e30a108c81126d4a9", - "zh:914d03e361a49fd296bafa7e10b0c228a5fb5e4f374078670f656166e8026700", - "zh:9749c9638c520e341726f981884d70f81025e368cb150a9b7cde7dc3f1f9c22b", + "h1:HuSOcwbYJMcq6kySrORAvi0HwPXuN9bqQLS2DyRWYVw=", + "zh:0237f82c4c5879f2830c70ff2f7b6bb0b574e3ad39c3f0c1d2f17b1caa194039", + "zh:10e8a6a5e216392b825dd017b0f61067970a57363c86fdb176dc4a2507322c16", + "zh:304e6252e002789c41f8b2eacd72842d8deb20bf31f2b88f3438a51ffd795346", + "zh:3cf9238ddec4030c564a7d0ba3ad2d3659f0369ceb0909e4b4b10593bd1a4f46", + "zh:5c57469081256d18309e31777cbd8529d1dacc1b92cc1df063afe612c8e1e0ca", + "zh:6502f3b6c6360d2b136d180b1889f2bb16578521406724d279c3403fc0550323", + "zh:739e3ce3059c03a0da57c8f63706b7bc5ea75c1164dba484623617e2b2d4ba76", + "zh:74f12c8e01b04bf01489e3a20fec9aa71d0171d9e8b4ca78afa5442ad55155ba", + "zh:9688e21395e21c57862b923fddd84115501e4660b65feec951bc412dc95005df", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9ceb0432160c11143e2556170f11c093ebbb088c2161a99eea105f6cb0c7e26a", - "zh:b7289a754153995187c887012f35c010bb8b23aed14bd5806c43ecc51602e266", - "zh:c5e81ed93f94361d8edc528250353f51e842e16ea1731d98919349b7bb30bd27", - "zh:e38b7a6d0b10fd01d6234c7e2c3f7595df791ea96c1f57ee24294f8758ee8fa6", - "zh:e3b6dbf42223d9f87f12345f74996932c56ae941fa4186ae7f7a1f3695284b4f", + "zh:a2f129f700bd37e674bbb85757675596e77969f09352d6c0c000ff66cf9e85c6", + "zh:b2dcbc45b43a463ea515442355e7f196eb9e9372ff55bea0e0cf211d121b867b", + "zh:bfac52745ddbc8e9ac08c4e1308ee49728965d6685f09819903f6fbad0e6bef3", + "zh:cbff07afb9a3e4f9ddb57c9fc42dae4db3ec80feb8119fe2be334da73286af4e", + "zh:d04f4c02720438136dde746ac942949fa1ebfe094239554eef627f455c997845", ] } diff --git a/iac/oci-prow-worker/terraform.tf b/iac/oci-prow-worker/terraform.tf index 3f3bc24..9688095 100644 --- a/iac/oci-prow-worker/terraform.tf +++ b/iac/oci-prow-worker/terraform.tf @@ -2,7 +2,7 @@ terraform { required_providers { oci = { source = "oracle/oci" - version = "5.36.0" + version = "6.2.0" } } From 65ea17d4006e775d19a388aebee72b63ea2a2e90 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 16 Jul 2024 16:19:45 +0200 Subject: [PATCH 09/10] Add .env.example file Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/.env.example | 3 +++ iac/oci-prow-worker/README.md | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 iac/oci-prow-worker/.env.example diff --git a/iac/oci-prow-worker/.env.example b/iac/oci-prow-worker/.env.example new file mode 100644 index 0000000..03e8e4b --- /dev/null +++ b/iac/oci-prow-worker/.env.example @@ -0,0 +1,3 @@ +export AWS_ACCESS_KEY_ID='' +export AWS_SECRET_ACCESS_KEY='' +export AWS_ENDPOINT_URL_S3='https://.compat.objectstorage.us-sanjose-1.oraclecloud.com' diff --git a/iac/oci-prow-worker/README.md b/iac/oci-prow-worker/README.md index 28c876a..6a47697 100644 --- a/iac/oci-prow-worker/README.md +++ b/iac/oci-prow-worker/README.md @@ -12,7 +12,7 @@ The following environment variables are required before running any `make` targe ## Running terraform -Easiest way to run terraform locally is to create a `.env` file with the required environment variables and then run `make` commands. For example: +Easiest way to run terraform locally is to create a `.env` file with the required environment variables and then run `make` commands. For example, create a file `.env` (or see [.env.example](./.env.example)): ```bash export AWS_ACCESS_KEY_ID=xxxxxxxxxxxxxxx @@ -34,4 +34,6 @@ oci_config_file_profile = "KCP" Install `oci` cli and run `oci session authenticate` to get the `oci_config_file` and `oci_profile` values. +Set up environment variables by running `source .env`. + Then run `make init` and `make plan` to see the changes that will be applied. If everything looks good, run `make apply`. From 755efb4a1425ed3b2782e3582f21bc75494f69ef Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 16 Jul 2024 16:20:22 +0200 Subject: [PATCH 10/10] Migrate cluster to use VCN native mode Signed-off-by: Marvin Beckers --- iac/oci-prow-worker/cluster.tf | 24 +++++++++++++++++++++++- iac/oci-prow-worker/network.tf | 20 +++++++++++++++----- iac/oci-prow-worker/outputs.tf | 6 ++++++ 3 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 iac/oci-prow-worker/outputs.tf diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf index 492d88e..1ccbc43 100644 --- a/iac/oci-prow-worker/cluster.tf +++ b/iac/oci-prow-worker/cluster.tf @@ -2,10 +2,27 @@ resource "oci_containerengine_cluster" "prow" { name = "oci-prow-worker" kubernetes_version = var.kubernetes_version + cluster_pod_network_options { + cni_type = "OCI_VCN_IP_NATIVE" + } + + endpoint_config { + is_public_ip_enabled = true + subnet_id = oci_core_subnet.prow_worker_cluster.id + } + + options { + service_lb_subnet_ids = [oci_core_subnet.prow_worker_cluster.id] + } + compartment_id = var.oci_compartment_ocid vcn_id = oci_core_vcn.prow.id } +data "oci_containerengine_cluster_kube_config" "prow" { + cluster_id = oci_containerengine_cluster.prow.id +} + resource "oci_containerengine_node_pool" "prow_worker" { cluster_id = oci_containerengine_cluster.prow.id compartment_id = var.oci_compartment_ocid @@ -39,8 +56,13 @@ resource "oci_containerengine_node_pool" "prow_worker" { for_each = data.oci_identity_availability_domains.availability_domains.availability_domains content { availability_domain = placement_configs.value.name - subnet_id = oci_core_subnet.prow_worker_cluster[placement_configs.key].id + subnet_id = oci_core_subnet.prow_worker_nodes.id } } + + node_pool_pod_network_option_details { + cni_type = "OCI_VCN_IP_NATIVE" + pod_subnet_ids = [oci_core_subnet.prow_worker_nodes.id] + } } } diff --git a/iac/oci-prow-worker/network.tf b/iac/oci-prow-worker/network.tf index 5a11175..223502d 100644 --- a/iac/oci-prow-worker/network.tf +++ b/iac/oci-prow-worker/network.tf @@ -22,15 +22,25 @@ resource "oci_core_route_table" "prow_worker" { } } -resource "oci_core_subnet" "prow_worker_cluster" { - count = length(data.oci_identity_availability_domains.availability_domains.availability_domains) +resource "oci_core_subnet" "prow_worker_nodes" { + availability_domain = null + cidr_block = "10.0.64.0/18" + compartment_id = var.oci_compartment_ocid + vcn_id = oci_core_vcn.prow.id + + security_list_ids = [oci_core_vcn.prow.default_security_list_id] + route_table_id = oci_core_route_table.prow_worker.id + display_name = "Prow Nodes/Pods Subnet" +} - availability_domain = data.oci_identity_availability_domains.availability_domains.availability_domains[count.index].name - cidr_block = "10.0.${20 + count.index}.0/24" +resource "oci_core_subnet" "prow_worker_cluster" { + availability_domain = null + cidr_block = "10.0.10.0/24" compartment_id = var.oci_compartment_ocid vcn_id = oci_core_vcn.prow.id security_list_ids = [oci_core_vcn.prow.default_security_list_id] route_table_id = oci_core_route_table.prow_worker.id - display_name = "Prow Cluster Subnet ${count.index}" + dhcp_options_id = oci_core_vcn.prow.default_dhcp_options_id + display_name = "Prow Cluster Subnet" } diff --git a/iac/oci-prow-worker/outputs.tf b/iac/oci-prow-worker/outputs.tf new file mode 100644 index 0000000..a9a8251 --- /dev/null +++ b/iac/oci-prow-worker/outputs.tf @@ -0,0 +1,6 @@ +output "cluster" { + value = { + kubeconfig = data.oci_containerengine_cluster_kube_config.prow.content + } + sensitive = true +}