Karpenter and other improvements

README.md

@@ -13,6 +13,7 @@ You will find latest setup of following components:
 1. VPC with public/private subnets, enabled flow logs and VPC endpoints for ECR and S3
 1. EKS controlplane
 1. EKS worker nodes in private subnets (spot and ondemnd instances based on variables)
+1. Karpenter configuration for nodes
 1. Option to used Managed Node Groups
 1. Dynamic basion host
 1. Automatically configure aws-auth configmap for worker nodes to join the cluster

terraform-aws/cluster-addons.tf

@@ -6,6 +6,8 @@ resource "aws_eks_addon" "kube_proxy" {
   addon_version     = var.eks_addon_version_kube_proxy
   resolve_conflicts = "OVERWRITE"
 
+  preserve = true
+
   tags = merge(
     var.tags,
     {
@@ -22,10 +24,65 @@ resource "aws_eks_addon" "core_dns" {
   addon_version     = var.eks_addon_version_core_dns
   resolve_conflicts = "OVERWRITE"
 
+  preserve = true
+
   tags = merge(
     var.tags,
     {
       "eks_addon" = "coredns"
     }
   )
 }
+
+resource "aws_eks_addon" "aws_ebs_csi_driver" {
+  count = alltrue([var.create_cluster, var.create_eks_addons]) ? 1 : 0
+
+  cluster_name      = aws_eks_cluster.cluster[0].name
+  addon_name        = "aws-ebs-csi-driver"
+  addon_version     = var.eks_addon_version_ebs_csi_driver
+  resolve_conflicts = "OVERWRITE"
+
+  service_account_role_arn = aws_iam_role.ebs_csi_controller_sa[0].arn
+
+  preserve = true
+
+  tags = merge(
+    var.tags,
+    {
+      "eks_addon" = "ebs-csi-driver"
+    }
+  )
+}
+
+resource "aws_iam_role" "ebs_csi_controller_sa" {
+  count = alltrue([var.create_cluster, var.create_eks_addons]) ? 1 : 0
+
+  name = "ebs-csi-controller-sa"
+
+  assume_role_policy = templatefile("policies/oidc_assume_role_policy.json", {
+    OIDC_ARN  = aws_iam_openid_connect_provider.cluster[0].arn,
+    OIDC_URL  = replace(aws_iam_openid_connect_provider.cluster[0].url, "https://", ""),
+    NAMESPACE = "kube-system",
+    SA_NAME   = "ebs-csi-controller-sa"
+  })
+
+  managed_policy_arns = ["arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"]
+}
+
+resource "aws_eks_addon" "kubecost" {
+  count = alltrue([var.create_cluster, var.create_eks_addons]) ? 1 : 0
+
+  cluster_name      = aws_eks_cluster.cluster[0].name
+  addon_name        = "kubecost_kubecost"
+  addon_version     = var.eks_addon_version_kubecost
+  resolve_conflicts = "OVERWRITE"
+
+  preserve = true
+
+  tags = merge(
+    var.tags,
+    {
+      "eks_addon" = "kubecost"
+    }
+  )
+}

terraform-aws/cluster.tf

@@ -20,11 +20,11 @@ resource "aws_security_group_rule" "vpc_endpoint_eks_cluster_sg" {
 # EKS Cluster
 #####
 module "kms-eks" {
-  source = "native-cube/kms/aws"
+  source  = "native-cube/kms/aws"
   version = "~> 1.0.0"
 
   alias_name = local.name_prefix
-  
+
   tags = var.tags
 }
 
@@ -62,7 +62,7 @@ resource "aws_eks_cluster" "cluster" {
 resource "aws_cloudwatch_log_group" "cluster" {
   name              = "/aws/eks/${local.name_prefix}/cluster"
   retention_in_days = 7
-  
+
   tags = var.tags
 }
 
@@ -102,7 +102,6 @@ resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSVPCResourceControlle
   role       = aws_iam_role.cluster.name
 }
 
-
 #####
 # Outputs
 #####

terraform-aws/data.tf

@@ -36,3 +36,50 @@ data "aws_iam_policy_document" "worker_node_role_assume_role_policy" {
     }
   }
 }
+
+data "aws_iam_policy_document" "eks_node_karpenter_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ec2.amazonaws.com"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "karpenter_controller" {
+  count = var.create_cluster ? 1 : 0
+
+  statement {
+    actions = [
+      "ec2:CreateLaunchTemplate",
+      "ec2:CreateFleet",
+      "ec2:RunInstances",
+      "ec2:CreateTags",
+      "ec2:TerminateInstances",
+      "ec2:DeleteLaunchTemplate",
+      "ec2:DescribeLaunchTemplates",
+      "ec2:DescribeInstances",
+      "ec2:DescribeSecurityGroups",
+      "ec2:DescribeSubnets",
+      "ec2:DescribeImages",
+      "ec2:DescribeInstanceTypes",
+      "ec2:DescribeInstanceTypeOfferings",
+      "ec2:DescribeAvailabilityZones",
+      "ec2:DescribeSpotPriceHistory",
+      "ssm:GetParameter",
+      "pricing:GetProducts"
+    ]
+
+    resources = ["*"]
+  }
+
+  statement {
+    actions = [
+      "iam:PassRole",
+    ]
+
+    resources = [aws_iam_role.eks_node_karpenter[0].arn]
+  }
+}

terraform-aws/eks-nodes-karpenter.tf

@@ -0,0 +1,131 @@
+#####
+# Karpenter Node IAM Role
+#####
+resource "aws_iam_role" "eks_node_karpenter" {
+  count = var.create_cluster ? 1 : 0
+
+  name = "${local.name_prefix}-node-karpenter"
+
+  assume_role_policy = data.aws_iam_policy_document.eks_node_karpenter_assume_role_policy.json
+
+  managed_policy_arns = [
+    "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
+    "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
+    "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
+    "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  ]
+}
+
+resource "aws_iam_instance_profile" "eks_node_karpenter" {
+  count = var.create_cluster ? 1 : 0
+
+  name = "${local.name_prefix}-node-karpenter"
+  role = aws_iam_role.eks_node_karpenter[0].name
+}
+
+# Used by karpenter-controller
+resource "aws_iam_role" "karpenter_controller" {
+  count = var.create_cluster ? 1 : 0
+
+  name        = "${local.name_prefix}-karpenter-controller"
+  description = "Allow karpenter-controller EC2 read and write operations."
+
+  assume_role_policy = templatefile("policies/oidc_assume_role_policy.json", {
+    OIDC_ARN  = aws_iam_openid_connect_provider.cluster[0].arn,
+    OIDC_URL  = replace(aws_iam_openid_connect_provider.cluster[0].url, "https://", ""),
+    NAMESPACE = "karpenter",
+    SA_NAME   = "karpenter"
+  })
+
+  force_detach_policies = true
+
+  tags = merge(
+    var.tags,
+    {
+      "ServiceAccountName"      = "karpenter"
+      "ServiceAccountNamespace" = "karpenter"
+    }
+  )
+}
+
+resource "aws_iam_role_policy" "karpenter_controller" {
+  count = var.create_cluster ? 1 : 0
+
+  name = "KarpenterControllerPolicy"
+  role = aws_iam_role.karpenter_controller[0].id
+
+  policy = data.aws_iam_policy_document.karpenter_controller[0].json
+}
+
+#####
+# Karpenter Node SG
+#####
+resource "aws_security_group" "node" {
+  count = var.create_cluster ? 1 : 0
+
+  name_prefix = "${local.name_prefix}-node-sg-"
+  description = "EKS Karpenter Node security group."
+  vpc_id      = module.vpc.vpc_id
+
+  tags = {
+    "Name"                                       = "${local.name_prefix}-node-sg"
+    "kubernetes.io/cluster/${local.name_prefix}" = "owned"
+    "karpenter.sh/discovery"                     = local.local.name_prefix
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_security_group_rule" "ingress_all_allow_access_from_control_plane" {
+  count = var.create_cluster ? 1 : 0
+
+  security_group_id = aws_security_group.node[0].id
+  description       = "Allow communication from control plan security group."
+
+  protocol                 = "-1"
+  from_port                = 0
+  to_port                  = 65535
+  source_security_group_id = aws_eks_cluster.cluster[0].vpc_config.0.cluster_security_group_id
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "ingress_self" {
+  count = var.create_cluster ? 1 : 0
+
+  security_group_id = aws_security_group.node[0].id
+  description       = "Self Reference all traffic"
+
+  protocol  = "-1"
+  from_port = 0
+  to_port   = 65535
+  type      = "ingress"
+  self      = true
+}
+
+resource "aws_security_group_rule" "egress_to_cluster" {
+  count = var.create_cluster ? 1 : 0
+
+  security_group_id = aws_security_group.node.id
+  description       = "Node groups to cluster API"
+
+  protocol                 = "-1"
+  from_port                = 0
+  to_port                  = 65535
+  type                     = "egress"
+  source_security_group_id = aws_eks_cluster.cluster[0].vpc_config.0.cluster_security_group_id
+}
+
+resource "aws_security_group_rule" "egress_to_internet" {
+  count = var.create_cluster ? 1 : 0
+
+  security_group_id = aws_security_group.node[0].id
+  description       = "All egress allowed."
+
+  from_port   = 0
+  to_port     = 65535
+  protocol    = "-1"
+  type        = "egress"
+  cidr_blocks = ["0.0.0.0/0"]
+}

terraform-aws/variables.tf

@@ -173,6 +173,19 @@ variable "eks_addon_version_core_dns" {
   default     = "v1.8.7-eksbuild.3"
 }
 
+variable "eks_addon_version_ebs_csi_driver" {
+  type        = string
+  description = "AWS ebs csi driver managed EKS addon version."
+  default     = "v1.13.0-eksbuild.3"
+}
+
+variable "eks_addon_version_kubecost" {
+  type        = string
+  description = "KubeCost EKS addon version."
+  default     = "v1.98.0-eksbuild.1"
+}
+
+
 variable "container_runtime" {
   type        = string
   description = "Container runtime used by EKS worker nodes. Allowed values: `dockerd` and `containerd`."

terraform-aws/versions.tf

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 4.26"
+      version = ">= 4.50"
     }
     random = {
       source  = "hashicorp/random"

terraform-aws/vpc.tf

@@ -16,7 +16,7 @@ resource "aws_default_security_group" "default" {
 
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
-  version = "3.14.2"
+  version = "3.18.1"
 
   name = "${local.name_prefix}-vpc"
 
@@ -145,7 +145,7 @@ resource "aws_security_group_rule" "vpc_endpoint_self_ingress" {
 #####
 
 module "vpc-flow-logs" {
-  source = "native-cube/vpc-flow-logs/aws"
+  source  = "native-cube/vpc-flow-logs/aws"
   version = "~> 1.0.0"
 
   name_prefix = local.name_prefix