From 1d1ff0b35aacb890354cf9d52775ad292bdb9588 Mon Sep 17 00:00:00 2001
From: Xi Bai <baixiac@gmail.com>
Date: Fri, 10 May 2024 14:32:50 +0100
Subject: [PATCH] make existing host zones importable and support EKS 1.28

---
 .github/workflows/cluster.yaml |  6 ++++--
 .github/workflows/config.yaml  |  6 ++++--
 .gitignore                     |  2 +-
 README.md                      |  2 +-
 cluster/data.tf                |  9 +++++++++
 cluster/eks.tf                 |  7 +++++++
 cluster/iam.tf                 |  6 ++----
 cluster/terraform.tfvars       |  2 +-
 cluster/variables.tf           |  6 +++---
 cluster/versions.tf            |  2 +-
 config/rds.tf                  |  7 +++++--
 config/route53.tf              | 37 +++++++++++++++++++++-------------
 config/ses.tf                  | 28 ++++++++++++-------------
 config/terraform.tfvars        |  4 ++--
 config/variables.tf            | 15 +++++++++-----
 config/versions.tf             |  2 +-
 16 files changed, 87 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/cluster.yaml b/.github/workflows/cluster.yaml
index ae30ac7..d3c3680 100644
--- a/.github/workflows/cluster.yaml
+++ b/.github/workflows/cluster.yaml
@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 env:
-  TERRAFORM_VERSION: 1.6.5
+  TERRAFORM_VERSION: 1.7.5
 
 jobs:
   build:
@@ -24,7 +24,9 @@ jobs:
           with:
             terraform_version: ${{ env.TERRAFORM_VERSION }}
         - name: Init
-          run: terraform init
+          run: |
+            rm -rf .terraform
+            terraform init -backend=false
         - name: Format
           run: terraform fmt -check
         - name: Validate
diff --git a/.github/workflows/config.yaml b/.github/workflows/config.yaml
index 0703379..9fa1ae3 100644
--- a/.github/workflows/config.yaml
+++ b/.github/workflows/config.yaml
@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 env:
-  TERRAFORM_VERSION: 1.6.5
+  TERRAFORM_VERSION: 1.7.5
 
 jobs:
   build:
@@ -24,7 +24,9 @@ jobs:
           with:
             terraform_version: ${{ env.TERRAFORM_VERSION }}
         - name: Init
-          run: terraform init
+          run: |
+            rm -rf .terraform
+            terraform init -backend=false
         - name: Format
           run: terraform fmt -check
         - name: Validate
diff --git a/.gitignore b/.gitignore
index 3b332d9..b44603b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
 **/.terraform
-**/.terraform.lock.hcl
+**/terraform.tfstate
 **/terraform.tfstate.backup
 misc/experimental
 .DS_Store
\ No newline at end of file
diff --git a/README.md b/README.md
index 82b5805..f264cb4 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ This repository aims to provide [IaC](https://en.wikipedia.org/wiki/Infrastructu
 [![Terraform validate](https://github.com/phidatalab/RADAR-K8s-Infrastructure/actions/workflows/config.yaml/badge.svg)](https://github.com/phidatalab/RADAR-K8s-Infrastructure/actions/workflows/config.yaml/badge.svg)
 
 # Dependencies
-[Terraform](https://developer.hashicorp.com/terraform/downloads) >= 1.4.0, < 1.7.0<br>
+[Terraform](https://developer.hashicorp.com/terraform/downloads) >= 1.7.0, < 1.8.0<br>
 [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) >= 2.11
 
 # Usage
diff --git a/cluster/data.tf b/cluster/data.tf
index f9cb771..11c26f0 100644
--- a/cluster/data.tf
+++ b/cluster/data.tf
@@ -1,6 +1,15 @@
 locals {
 
   eks_core_versions = {
+    "1.28" : {
+      "cluster_version" = "1.28"
+      "cluster_addons" = {
+        "coredns"        = "v1.10.1-eksbuild.10"
+        "kube_proxy"     = "v1.28.1-eksbuild.1"
+        "vpc_cni"        = "v1.16.4-eksbuild.2"
+        "ebs_csi_driver" = "v1.26.1-eksbuild.1"
+      }
+    },
     "1.27" : {
       "cluster_version" = "1.27"
       "cluster_addons" = {
diff --git a/cluster/eks.tf b/cluster/eks.tf
index 292f83c..a20eccb 100644
--- a/cluster/eks.tf
+++ b/cluster/eks.tf
@@ -213,6 +213,13 @@ module "eks" {
     },
   ]
 
+  kms_key_administrators = [
+    "arn:aws:iam::${module.vpc.vpc_owner_id}:root"
+  ]
+  kms_key_users = [
+    module.eks_admins_iam_role.iam_role_arn,
+  ]
+
   tags = merge(tomap({ "Name" : var.eks_cluster_name }), var.common_tags)
 }
 
diff --git a/cluster/iam.tf b/cluster/iam.tf
index 52b6a8c..fd35a9e 100644
--- a/cluster/iam.tf
+++ b/cluster/iam.tf
@@ -81,7 +81,7 @@ module "iam_user" {
 
   name                          = "${var.eks_cluster_name}-ecr-readonly-user"
   create_iam_user_login_profile = false
-  create_iam_access_key         = true
+  create_iam_access_key         = false
   force_destroy                 = false
   policy_arns = [
     "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
@@ -107,9 +107,7 @@ resource "aws_iam_policy" "s3_access" {
           "s3:DeleteObject"
         ]
         Resource = [
-          "arn:aws:s3:::${var.eks_cluster_name}-intermediate-output-storage/*",
-          "arn:aws:s3:::${var.eks_cluster_name}-output-storage/*",
-          "arn:aws:s3:::${var.eks_cluster_name}-velero-backups/*",
+          "arn:aws:s3:::${var.eks_cluster_name}-*/*",
         ]
       }
     ]
diff --git a/cluster/terraform.tfvars b/cluster/terraform.tfvars
index ecd08be..c3bd698 100644
--- a/cluster/terraform.tfvars
+++ b/cluster/terraform.tfvars
@@ -1,5 +1,5 @@
 AWS_REGION             = "eu-west-2"
-eks_kubernetes_version = "1.27"
+eks_kubernetes_version = "1.28"
 environment            = "dev"
 eks_admins_group_users = []
 defaut_storage_class   = "radar-base-ebs-sc-gp2"
diff --git a/cluster/variables.tf b/cluster/variables.tf
index cd3da69..7d710f4 100644
--- a/cluster/variables.tf
+++ b/cluster/variables.tf
@@ -51,11 +51,11 @@ variable "common_tags" {
 variable "eks_kubernetes_version" {
   type        = string
   description = "Amazon EKS Kubernetes version"
-  default     = "1.27"
+  default     = "1.28"
 
   validation {
-    condition     = contains(["1.27", "1.26", "1.25"], var.eks_kubernetes_version)
-    error_message = "Invalid EKS Kubernetes version. Supported versions are  '1.27', '1.26', '1.25'."
+    condition     = contains(["1.28", "1.27", "1.26", "1.25"], var.eks_kubernetes_version)
+    error_message = "Invalid EKS Kubernetes version. Supported versions are '1.28', '1.27', '1.26', '1.25'."
   }
 }
 
diff --git a/cluster/versions.tf b/cluster/versions.tf
index 5c944e7..f97579b 100644
--- a/cluster/versions.tf
+++ b/cluster/versions.tf
@@ -13,5 +13,5 @@ terraform {
       version = "~> 1.14.0"
     }
   }
-  required_version = ">= 1.4.0, < 1.7.0"
+  required_version = ">= 1.7.0, < 1.8.0"
 }
diff --git a/config/rds.tf b/config/rds.tf
index e03e79b..4a0f9d7 100644
--- a/config/rds.tf
+++ b/config/rds.tf
@@ -60,14 +60,14 @@ resource "aws_db_instance" "radar_postgres" {
   tags = merge(tomap({ "Name" : "${var.eks_cluster_name}-postgres" }), var.common_tags)
 }
 
-resource "kubectl_manifest" "create_databases" {
+resource "kubectl_manifest" "create_databases_if_not_exist" {
   count = var.enable_rds ? 1 : 0
 
   yaml_body = <<-YAML
     apiVersion: batch/v1
     kind: Job
     metadata:
-      name: create-radar-postgres-databases
+      name: create-radar-postgres-databases-if-not-exist
     spec:
       template:
         spec:
@@ -81,7 +81,10 @@ resource "kubectl_manifest" "create_databases" {
                   PGPASSWORD=${var.radar_postgres_password} psql --host=${aws_db_instance.radar_postgres[0].address} --port=5432 --username=${aws_db_instance.radar_postgres[0].username} --dbname=radarbase -c 'CREATE DATABASE managementportal;'
                   PGPASSWORD=${var.radar_postgres_password} psql --host=${aws_db_instance.radar_postgres[0].address} --port=5432 --username=${aws_db_instance.radar_postgres[0].username} --dbname=radarbase -c 'CREATE DATABASE appserver;'
                   PGPASSWORD=${var.radar_postgres_password} psql --host=${aws_db_instance.radar_postgres[0].address} --port=5432 --username=${aws_db_instance.radar_postgres[0].username} --dbname=radarbase -c 'CREATE DATABASE rest_sources_auth;'
+                  true
           restartPolicy: Never
+      activeDeadlineSeconds: 60
+      ttlSecondsAfterFinished: 60
   YAML
 
   depends_on = [
diff --git a/config/route53.tf b/config/route53.tf
index b201089..f9a110e 100644
--- a/config/route53.tf
+++ b/config/route53.tf
@@ -1,39 +1,48 @@
-resource "aws_route53_zone" "primary" {
-  count = var.enable_route53 ? 1 : 0
+locals {
+  domain_name = length(var.domain_name) == 0 ? null : keys(var.domain_name)[0]
+}
 
-  name = var.domain_name
-  tags = merge(tomap({ "Name" : "${var.eks_cluster_name}-primary-zone" }), var.common_tags)
+import {
+  for_each = var.domain_name
+  to       = aws_route53_zone.primary[0]
+  id       = each.value
+}
+
+resource "aws_route53_zone" "primary" {
+  count = var.enable_route53 && length(var.domain_name) == 1 ? 1 : 0
+  name  = local.domain_name
+  tags  = merge(tomap({ "Name" : "${var.eks_cluster_name}-primary-zone" }), var.common_tags)
 }
 
 resource "aws_route53_record" "main" {
-  count = var.enable_route53 && var.enable_eip ? 1 : 0
+  count = var.enable_route53 && length(var.domain_name) == 1 && var.enable_eip ? 1 : 0
 
   zone_id = aws_route53_zone.primary[0].zone_id
-  name    = "${var.environment}.${var.domain_name}"
+  name    = "${var.environment}.${local.domain_name}"
   type    = "CNAME"
   ttl     = 300
   records = [aws_eip.cluster_loadbalancer_eip[0].public_dns]
 }
 
 resource "aws_route53_record" "this" {
-  for_each = toset([for prefix in local.cname_prefixes : prefix if var.enable_route53])
+  for_each = toset([for prefix in local.cname_prefixes : prefix if var.enable_route53 && length(var.domain_name) == 1])
 
   zone_id = aws_route53_zone.primary[0].zone_id
-  name    = "${each.value}.${var.environment}.${var.domain_name}"
+  name    = "${each.value}.${var.environment}.${local.domain_name}"
   type    = "CNAME"
   ttl     = 300
-  records = ["${var.environment}.${var.domain_name}"]
+  records = ["${var.environment}.${local.domain_name}"]
 }
 
 module "external_dns_irsa" {
-  count = var.enable_route53 ? 1 : 0
+  count = var.enable_route53 && length(var.domain_name) == 1 ? 1 : 0
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
   version = "~> 5.0"
 
   role_name                     = "${var.eks_cluster_name}-external-dns-irsa"
   attach_external_dns_policy    = true
-  external_dns_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${aws_route53_zone.primary[0].id}"]
+  external_dns_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${aws_route53_zone.primary[0].zone_id}"]
 
   oidc_providers = {
     ex = {
@@ -46,14 +55,14 @@ module "external_dns_irsa" {
 }
 
 module "cert_manager_irsa" {
-  count = var.enable_route53 ? 1 : 0
+  count = var.enable_route53 && length(var.domain_name) == 1 ? 1 : 0
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
   version = "~> 5.0"
 
   role_name                     = "${var.eks_cluster_name}-cert-manager-irsa"
   attach_cert_manager_policy    = true
-  cert_manager_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${aws_route53_zone.primary[0].id}"]
+  cert_manager_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${aws_route53_zone.primary[0].zone_id}"]
 
   oidc_providers = {
     main = {
@@ -66,5 +75,5 @@ module "cert_manager_irsa" {
 }
 
 output "radar_base_route53_hosted_zone_id" {
-  value = var.enable_route53 ? aws_route53_zone.primary[0].zone_id : null
+  value = var.enable_route53 && length(var.domain_name) == 1 ? aws_route53_zone.primary[0].zone_id : null
 }
diff --git a/config/ses.tf b/config/ses.tf
index 471165a..e4eff10 100644
--- a/config/ses.tf
+++ b/config/ses.tf
@@ -1,53 +1,51 @@
 resource "aws_ses_domain_identity" "smtp_identity" {
-  count = var.enable_ses ? 1 : 0
-
-  domain = var.domain_name
+  count  = var.enable_route53 && length(var.domain_name) == 1 && var.enable_ses ? 1 : 0
+  domain = keys(var.domain_name)[0]
 }
 
 resource "aws_ses_domain_dkim" "smtp_dkim" {
-  count = var.enable_ses ? 1 : 0
-
+  count  = var.enable_route53 && length(var.domain_name) == 1 && var.enable_ses ? 1 : 0
   domain = aws_ses_domain_identity.smtp_identity[0].domain
 }
 
 resource "aws_route53_record" "smtp_dkim_record" {
-  count   = var.enable_route53 && var.enable_ses ? 3 : 0
-  zone_id = aws_route53_zone.primary[0].id
+  count   = var.enable_route53 && length(var.domain_name) == 1 && var.enable_ses ? 3 : 0
+  zone_id = aws_route53_zone.primary[0].zone_id
   name    = "${aws_ses_domain_dkim.smtp_dkim[0].dkim_tokens[count.index]}._domainkey"
   type    = "CNAME"
   ttl     = "600"
   records = ["${aws_ses_domain_dkim.smtp_dkim[0].dkim_tokens[count.index]}.dkim.amazonses.com"]
 
-  depends_on = [aws_route53_zone.primary[0]]
+  depends_on = [aws_route53_zone.primary]
 }
 
 resource "aws_ses_domain_mail_from" "smtp_mail_from" {
-  count = var.enable_ses ? 1 : 0
+  count = var.enable_route53 && length(var.domain_name) == 1 && var.enable_ses ? 1 : 0
 
   domain           = aws_ses_domain_identity.smtp_identity[0].domain
   mail_from_domain = "info.${var.environment}.${aws_ses_domain_identity.smtp_identity[0].domain}"
 }
 
 resource "aws_route53_record" "smtp_mail_from_mx" {
-  count   = var.enable_route53 && var.enable_ses ? 1 : 0
-  zone_id = aws_route53_zone.primary[0].id
+  count   = var.enable_route53 && length(var.domain_name) == 1 && var.enable_ses ? 1 : 0
+  zone_id = aws_route53_zone.primary[0].zone_id
   name    = aws_ses_domain_mail_from.smtp_mail_from[0].mail_from_domain
   type    = "MX"
   ttl     = "600"
   records = ["10 feedback-smtp.${var.AWS_REGION}.amazonses.com"]
 
-  depends_on = [aws_route53_zone.primary[0]]
+  depends_on = [aws_route53_zone.primary]
 }
 
 resource "aws_route53_record" "smtp_mail_from_txt" {
-  count   = var.enable_route53 && var.enable_ses ? 1 : 0
-  zone_id = aws_route53_zone.primary[0].id
+  count   = var.enable_route53 && length(var.domain_name) == 1 && var.enable_ses ? 1 : 0
+  zone_id = aws_route53_zone.primary[0].zone_id
   name    = aws_ses_domain_mail_from.smtp_mail_from[0].mail_from_domain
   type    = "TXT"
   ttl     = "600"
   records = ["v=spf1 include:amazonses.com ~all"]
 
-  depends_on = [aws_route53_zone.primary[0]]
+  depends_on = [aws_route53_zone.primary]
 }
 
 resource "aws_iam_user" "smtp_user" {
diff --git a/config/terraform.tfvars b/config/terraform.tfvars
index 71ca455..0082eb4 100644
--- a/config/terraform.tfvars
+++ b/config/terraform.tfvars
@@ -1,6 +1,6 @@
 AWS_REGION       = "eu-west-2"
 environment      = "dev"
-domain_name      = "change-me-radar-base-dummy-domain.net"
+domain_name      = {} # Pair of top level domain and hosted zone ID for deployed applications
 with_dmz_pods    = false
 enable_karpenter = false
 enable_msk       = false
@@ -8,4 +8,4 @@ enable_rds       = false
 enable_route53   = false
 enable_ses       = false
 enable_s3        = false
-enable_eip       = false
\ No newline at end of file
+enable_eip       = false
diff --git a/config/variables.tf b/config/variables.tf
index f71dc31..be23cd0 100644
--- a/config/variables.tf
+++ b/config/variables.tf
@@ -43,15 +43,20 @@ variable "common_tags" {
   type        = map(string)
   description = "Common tags associated to resources created"
   default = {
-    Project     = "radar-base-development"
+    Project     = "radar-base"
     Environment = "dev"
   }
 }
 
 variable "domain_name" {
-  type        = string
-  description = "Top level domain for deployed applications"
-  default     = "change-me-radar-base-dummy-domain.net"
+  type        = map(string)
+  description = "Pair of top level domain and hosted zone ID for deployed applications"
+  default     = {}
+
+  validation {
+    condition     = length(var.domain_name) < 2
+    error_message = "Multiple domain and hosted zone pairs are not supported."
+  }
 }
 
 variable "instance_capacity_type" {
@@ -72,7 +77,7 @@ variable "kafka_version" {
 
 variable "postgres_version" {
   type    = string
-  default = "13.7"
+  default = "13.14"
 
 }
 
diff --git a/config/versions.tf b/config/versions.tf
index 9b313dc..8d2a065 100644
--- a/config/versions.tf
+++ b/config/versions.tf
@@ -17,5 +17,5 @@ terraform {
       version = "~> 1.14.0"
     }
   }
-  required_version = ">= 1.4.0, < 1.7.0"
+  required_version = ">= 1.7.0, < 1.8.0"
 }