diff --git a/iac/oci-prow-worker/.env.example b/iac/oci-prow-worker/.env.example new file mode 100644 index 0000000..03e8e4b --- /dev/null +++ b/iac/oci-prow-worker/.env.example @@ -0,0 +1,3 @@ +export AWS_ACCESS_KEY_ID='' +export AWS_SECRET_ACCESS_KEY='' +export AWS_ENDPOINT_URL_S3='https://.compat.objectstorage.us-sanjose-1.oraclecloud.com' diff --git a/iac/oci-prow-worker/.gitignore b/iac/oci-prow-worker/.gitignore new file mode 100644 index 0000000..cc49b2f --- /dev/null +++ b/iac/oci-prow-worker/.gitignore @@ -0,0 +1,6 @@ +# Terraform folder +.terraform +# Make sure to not allow checking in tfvars by mistake +*.tfvars +# Environment variables are often stored in this file +.env diff --git a/iac/oci-prow-worker/.terraform.lock.hcl b/iac/oci-prow-worker/.terraform.lock.hcl new file mode 100644 index 0000000..a237fab --- /dev/null +++ b/iac/oci-prow-worker/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/oracle/oci" { + version = "6.2.0" + constraints = "6.2.0" + hashes = [ + "h1:HuSOcwbYJMcq6kySrORAvi0HwPXuN9bqQLS2DyRWYVw=", + "zh:0237f82c4c5879f2830c70ff2f7b6bb0b574e3ad39c3f0c1d2f17b1caa194039", + "zh:10e8a6a5e216392b825dd017b0f61067970a57363c86fdb176dc4a2507322c16", + "zh:304e6252e002789c41f8b2eacd72842d8deb20bf31f2b88f3438a51ffd795346", + "zh:3cf9238ddec4030c564a7d0ba3ad2d3659f0369ceb0909e4b4b10593bd1a4f46", + "zh:5c57469081256d18309e31777cbd8529d1dacc1b92cc1df063afe612c8e1e0ca", + "zh:6502f3b6c6360d2b136d180b1889f2bb16578521406724d279c3403fc0550323", + "zh:739e3ce3059c03a0da57c8f63706b7bc5ea75c1164dba484623617e2b2d4ba76", + "zh:74f12c8e01b04bf01489e3a20fec9aa71d0171d9e8b4ca78afa5442ad55155ba", + "zh:9688e21395e21c57862b923fddd84115501e4660b65feec951bc412dc95005df", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a2f129f700bd37e674bbb85757675596e77969f09352d6c0c000ff66cf9e85c6", + "zh:b2dcbc45b43a463ea515442355e7f196eb9e9372ff55bea0e0cf211d121b867b", + "zh:bfac52745ddbc8e9ac08c4e1308ee49728965d6685f09819903f6fbad0e6bef3", + "zh:cbff07afb9a3e4f9ddb57c9fc42dae4db3ec80feb8119fe2be334da73286af4e", + "zh:d04f4c02720438136dde746ac942949fa1ebfe094239554eef627f455c997845", + ] +} diff --git a/iac/oci-prow-worker/Makefile b/iac/oci-prow-worker/Makefile new file mode 100644 index 0000000..5b19532 --- /dev/null +++ b/iac/oci-prow-worker/Makefile @@ -0,0 +1,13 @@ +OPENTOFU_CLI ?= tofu + +init: + $(OPENTOFU_CLI) init + +fmt: + $(OPENTOFU_CLI) fmt + +plan: + $(OPENTOFU_CLI) plan + +apply: + $(OPENTOFU_CLI) apply diff --git a/iac/oci-prow-worker/README.md b/iac/oci-prow-worker/README.md new file mode 100644 index 0000000..6a47697 --- /dev/null +++ b/iac/oci-prow-worker/README.md @@ -0,0 +1,39 @@ +# oci-prow-cluster + +This directory deploys the `oci-prow-cluster` OKE cluster in OCI (Oracle Cloud) via [OpenTofu](https://opentofu.org). A shared state is stored in a OCI storage bucket, please make sure to use that. Usually, this code shouldn't be executed directly but run by Prow. + +## Required Environment Variables + +The following environment variables are required before running any `make` targets: + +- `AWS_ACCESS_KEY_ID`: Needs to be the key ID for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets. +- `AWS_SECRET_ACCESS_KEY`: Needs to be the secret for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets. +- `AWS_ENDPOINT_URL_S3`: Needs to be `https://.compat.objectstorage.us-sanjose-1.oraclecloud.com`. Replace `` with the namespace displayed on the bucket (see OCI Console for this information). + +## Running terraform + +Easiest way to run terraform locally is to create a `.env` file with the required environment variables and then run `make` commands. For example, create a file `.env` (or see [.env.example](./.env.example)): + +```bash +export AWS_ACCESS_KEY_ID=xxxxxxxxxxxxxxx +export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxx +export AWS_ENDPOINT_URL_S3=https://xxxxxxxxxxxx.compat.objectstorage.us-sanjose-1.oraclecloud.com +export TF_LOG=DEBUG +``` + +Create `terraform.tfvars` file with the following content: + +```hcl +oci_tenant_ocid = "ocid1.tenancy.oc1..xxxxxxxxxxxxxxxxxxx" +oci_compartment_ocid = "ocid1.compartment.oc1..xxxxxxxxxxxxxxxxxxx" +oci_region = "us-sanjose-1" +node_pool_ssh_public_key = "ssh-rsa " +oci_auth_type = "SecurityToken" +oci_config_file_profile = "KCP" +``` + +Install `oci` cli and run `oci session authenticate` to get the `oci_config_file` and `oci_profile` values. + +Set up environment variables by running `source .env`. + +Then run `make init` and `make plan` to see the changes that will be applied. If everything looks good, run `make apply`. diff --git a/iac/oci-prow-worker/cluster.tf b/iac/oci-prow-worker/cluster.tf new file mode 100644 index 0000000..1ccbc43 --- /dev/null +++ b/iac/oci-prow-worker/cluster.tf @@ -0,0 +1,68 @@ +resource "oci_containerengine_cluster" "prow" { + name = "oci-prow-worker" + kubernetes_version = var.kubernetes_version + + cluster_pod_network_options { + cni_type = "OCI_VCN_IP_NATIVE" + } + + endpoint_config { + is_public_ip_enabled = true + subnet_id = oci_core_subnet.prow_worker_cluster.id + } + + options { + service_lb_subnet_ids = [oci_core_subnet.prow_worker_cluster.id] + } + + compartment_id = var.oci_compartment_ocid + vcn_id = oci_core_vcn.prow.id +} + +data "oci_containerengine_cluster_kube_config" "prow" { + cluster_id = oci_containerengine_cluster.prow.id +} + +resource "oci_containerengine_node_pool" "prow_worker" { + cluster_id = oci_containerengine_cluster.prow.id + compartment_id = var.oci_compartment_ocid + + kubernetes_version = var.kubernetes_version + name = "prow-worker" + ssh_public_key = var.node_pool_ssh_public_key + + # this matches t3.2xlarge sizings. + node_shape = "VM.Standard.A1.Flex" + node_shape_config { + memory_in_gbs = 32 + ocpus = 8 + } + + + # Using image Oracle-Linux-7.x- + # Find image OCID for your region from https://docs.oracle.com/iaas/images/ + # For now aarch64 lates k/k 1.29 image is used. + node_source_details { + image_id = "ocid1.image.oc1.us-sanjose-1.aaaaaaaaceb5egr4du2d5vut6uam2kdbctilom4w5wirnz7tihe4w4y3yroq" + source_type = "image" + } + + node_config_details { + size = var.node_pool_worker_size + + # create placement_configs for each availability domain. + # There happens to be only a single one in us-sanjose-1. + dynamic "placement_configs" { + for_each = data.oci_identity_availability_domains.availability_domains.availability_domains + content { + availability_domain = placement_configs.value.name + subnet_id = oci_core_subnet.prow_worker_nodes.id + } + } + + node_pool_pod_network_option_details { + cni_type = "OCI_VCN_IP_NATIVE" + pod_subnet_ids = [oci_core_subnet.prow_worker_nodes.id] + } + } +} diff --git a/iac/oci-prow-worker/network.tf b/iac/oci-prow-worker/network.tf new file mode 100644 index 0000000..223502d --- /dev/null +++ b/iac/oci-prow-worker/network.tf @@ -0,0 +1,46 @@ +resource "oci_core_vcn" "prow" { + cidr_block = "10.0.0.0/16" + compartment_id = var.oci_compartment_ocid + display_name = "Prow Network" +} + +resource "oci_core_internet_gateway" "prow" { + compartment_id = var.oci_compartment_ocid + display_name = "Prow Internet Gateway" + vcn_id = oci_core_vcn.prow.id +} + +resource "oci_core_route_table" "prow_worker" { + compartment_id = var.oci_compartment_ocid + vcn_id = oci_core_vcn.prow.id + display_name = "Prow Worker Route Table" + + route_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_internet_gateway.prow.id + } +} + +resource "oci_core_subnet" "prow_worker_nodes" { + availability_domain = null + cidr_block = "10.0.64.0/18" + compartment_id = var.oci_compartment_ocid + vcn_id = oci_core_vcn.prow.id + + security_list_ids = [oci_core_vcn.prow.default_security_list_id] + route_table_id = oci_core_route_table.prow_worker.id + display_name = "Prow Nodes/Pods Subnet" +} + +resource "oci_core_subnet" "prow_worker_cluster" { + availability_domain = null + cidr_block = "10.0.10.0/24" + compartment_id = var.oci_compartment_ocid + vcn_id = oci_core_vcn.prow.id + + security_list_ids = [oci_core_vcn.prow.default_security_list_id] + route_table_id = oci_core_route_table.prow_worker.id + dhcp_options_id = oci_core_vcn.prow.default_dhcp_options_id + display_name = "Prow Cluster Subnet" +} diff --git a/iac/oci-prow-worker/outputs.tf b/iac/oci-prow-worker/outputs.tf new file mode 100644 index 0000000..a9a8251 --- /dev/null +++ b/iac/oci-prow-worker/outputs.tf @@ -0,0 +1,6 @@ +output "cluster" { + value = { + kubeconfig = data.oci_containerengine_cluster_kube_config.prow.content + } + sensitive = true +} diff --git a/iac/oci-prow-worker/provider.tf b/iac/oci-prow-worker/provider.tf new file mode 100644 index 0000000..7ceb658 --- /dev/null +++ b/iac/oci-prow-worker/provider.tf @@ -0,0 +1,11 @@ +provider "oci" { + tenancy_ocid = var.oci_tenant_ocid + region = var.oci_region + auth = var.oci_auth_type + config_file_profile = var.oci_config_file_profile +} + +data "oci_identity_availability_domains" "availability_domains" { + compartment_id = var.oci_tenant_ocid +} + diff --git a/iac/oci-prow-worker/terraform.tf b/iac/oci-prow-worker/terraform.tf new file mode 100644 index 0000000..9688095 --- /dev/null +++ b/iac/oci-prow-worker/terraform.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + oci = { + source = "oracle/oci" + version = "6.2.0" + } + } + + # make sure to set AWS_ENDPOINT_URL_S3 to 'https://.compat.objectstorage.us-sanjose-1.oraclecloud.com'. + backend "s3" { + bucket = "kcp-opentofu-state" + region = "us-sanjose-1" + key = "ci-prow-worker/tf.tfstate" + + skip_region_validation = true + skip_credentials_validation = true + skip_requesting_account_id = true + use_path_style = true + skip_metadata_api_check = true + } +} diff --git a/iac/oci-prow-worker/variables.tf b/iac/oci-prow-worker/variables.tf new file mode 100644 index 0000000..d22ec6d --- /dev/null +++ b/iac/oci-prow-worker/variables.tf @@ -0,0 +1,46 @@ +variable "oci_tenant_ocid" { + type = string +} + +variable "oci_compartment_ocid" { + type = string +} + +/* +variable "oci_user_ocid" { + type = string +} + +variable "oci_private_key" { + type = string + sensitive = true +} +*/ + +variable "oci_region" { + type = string +} + +variable "node_pool_ssh_public_key" { + type = string +} + +variable "node_pool_worker_size" { + type = number + default = 3 +} + +variable "kubernetes_version" { + type = string + default = "v1.29.1" +} + +variable "oci_config_file_profile" { + type = string + default = "DEFAULT" +} + +variable "oci_auth_type" { + type = string + default = "SecurityToken" +}