Implement mutual TLS authentication support for Ingress (#3532)

* Implement mutual TLS authentication support for Ingress

* Addressing comments and adding policy updates

* Adding dependency for maps

* removing ingressClassParam support

.github/workflows/test.yaml

@@ -10,7 +10,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.19
+        go-version: ^1.21
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2

controllers/ingress/group_controller.go

@@ -56,7 +56,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
 	trackingProvider := tracking.NewDefaultProvider(ingressTagPrefix, controllerConfig.ClusterName)
 	elbv2TaggingManager := elbv2deploy.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, cloud.RGT(), logger)
 	modelBuilder := ingress.NewDefaultModelBuilder(k8sClient, eventRecorder,
-		cloud.EC2(), cloud.ACM(),
+		cloud.EC2(), cloud.ELBV2(), cloud.ACM(),
 		annotationParser, subnetsResolver,
 		authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates,
 		cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,

docs/install/iam_policy.json

@@ -38,7 +38,8 @@
                 "elasticloadbalancing:DescribeTargetGroups",
                 "elasticloadbalancing:DescribeTargetGroupAttributes",
                 "elasticloadbalancing:DescribeTargetHealth",
-                "elasticloadbalancing:DescribeTags"
+                "elasticloadbalancing:DescribeTags",
+                "elasticloadbalancing:DescribeTrustStores"
             ],
             "Resource": "*"
         },

docs/install/iam_policy_us-gov.json

@@ -38,7 +38,8 @@
                 "elasticloadbalancing:DescribeTargetGroups",
                 "elasticloadbalancing:DescribeTargetGroupAttributes",
                 "elasticloadbalancing:DescribeTargetHealth",
-                "elasticloadbalancing:DescribeTags"
+                "elasticloadbalancing:DescribeTags",
+                "elasticloadbalancing:DescribeTrustStores"
             ],
             "Resource": "*"
         },

go.mod

@@ -1,9 +1,9 @@
 module sigs.k8s.io/aws-load-balancer-controller
 
-go 1.20
+go 1.21
 
 require (
-	github.com/aws/aws-sdk-go v1.47.13
+	github.com/aws/aws-sdk-go v1.48.10
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/gavv/httpexpect/v2 v2.9.0
 	github.com/go-logr/logr v1.2.4
@@ -23,6 +23,7 @@ require (
 	k8s.io/apimachinery v0.26.5
 	k8s.io/cli-runtime v0.26.3
 	k8s.io/client-go v0.26.5
+	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
 	sigs.k8s.io/controller-runtime v0.14.6
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -156,7 +157,6 @@ require (
 	k8s.io/klog/v2 v2.80.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	k8s.io/kubectl v0.26.0 // indirect
-	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	oras.land/oras-go v1.2.2 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect

go.sum

@@ -79,6 +79,8 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/aws/aws-sdk-go v1.47.13 h1:pJgCtldg5azDAFoEcE0fz6n+FnCc1/FY4krtUa5uvZQ=
 github.com/aws/aws-sdk-go v1.47.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.48.10 h1:0LIFG3wp2Dt6PsxKWCg1Y1xRrn2vZnW5/gWdgaBalKg=
+github.com/aws/aws-sdk-go v1.48.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

pkg/annotations/constants.go

@@ -46,6 +46,7 @@ const (
 	IngressSuffixAuthSessionTimeout           = "auth-session-timeout"
 	IngressSuffixTargetNodeLabels             = "target-node-labels"
 	IngressSuffixManageSecurityGroupRules     = "manage-backend-security-group-rules"
+	IngressSuffixMutualAuthentication         = "mutual-authentication"
 
 	// NLB annotation suffixes
 	// prefixes service.beta.kubernetes.io, service.kubernetes.io

pkg/deploy/elbv2/listener_manager.go

@@ -2,6 +2,7 @@ package elbv2
 
 import (
 	"context"
+	"reflect"
 	"time"
 
 	awssdk "github.com/aws/aws-sdk-go/aws"
@@ -134,7 +135,8 @@ func (m *defaultListenerManager) updateSDKListenerWithSettings(ctx context.Conte
 		return err
 	}
 	desiredDefaultCerts, _ := buildSDKCertificates(resLS.Spec.Certificates)
-	if !isSDKListenerSettingsDrifted(resLS.Spec, sdkLS, desiredDefaultActions, desiredDefaultCerts) {
+	desiredDefaultMutualAuthentication := buildSDKMutualAuthenticationConfig(resLS.Spec.MutualAuthentication)
+	if !isSDKListenerSettingsDrifted(resLS.Spec, sdkLS, desiredDefaultActions, desiredDefaultCerts, desiredDefaultMutualAuthentication) {
 		return nil
 	}
 	req := buildSDKModifyListenerInput(resLS.Spec, desiredDefaultActions, desiredDefaultCerts)
@@ -246,7 +248,7 @@ func (m *defaultListenerManager) fetchSDKListenerExtraCertificateARNs(ctx contex
 }
 
 func isSDKListenerSettingsDrifted(lsSpec elbv2model.ListenerSpec, sdkLS ListenerWithTags,
-	desiredDefaultActions []*elbv2sdk.Action, desiredDefaultCerts []*elbv2sdk.Certificate) bool {
+	desiredDefaultActions []*elbv2sdk.Action, desiredDefaultCerts []*elbv2sdk.Certificate, desiredDefaultMutualAuthentication *elbv2sdk.MutualAuthenticationAttributes) bool {
 	if lsSpec.Port != awssdk.Int64Value(sdkLS.Listener.Port) {
 		return true
 	}
@@ -265,6 +267,9 @@ func isSDKListenerSettingsDrifted(lsSpec elbv2model.ListenerSpec, sdkLS Listener
 	if len(lsSpec.ALPNPolicy) != 0 && !cmp.Equal(lsSpec.ALPNPolicy, awssdk.StringValueSlice(sdkLS.Listener.AlpnPolicy), cmpopts.EquateEmpty()) {
 		return true
 	}
+	if !reflect.DeepEqual(desiredDefaultMutualAuthentication, sdkLS.Listener.MutualAuthentication) {
+		return true
+	}
 
 	return false
 }
@@ -289,6 +294,8 @@ func buildSDKCreateListenerInput(lsSpec elbv2model.ListenerSpec, featureGates co
 	if len(lsSpec.ALPNPolicy) != 0 {
 		sdkObj.AlpnPolicy = awssdk.StringSlice(lsSpec.ALPNPolicy)
 	}
+	sdkObj.MutualAuthentication = buildSDKMutualAuthenticationConfig(lsSpec.MutualAuthentication)
+
 	return sdkObj, nil
 }
 
@@ -302,6 +309,8 @@ func buildSDKModifyListenerInput(lsSpec elbv2model.ListenerSpec, desiredDefaultA
 	if len(lsSpec.ALPNPolicy) != 0 {
 		sdkObj.AlpnPolicy = awssdk.StringSlice(lsSpec.ALPNPolicy)
 	}
+	sdkObj.MutualAuthentication = buildSDKMutualAuthenticationConfig(lsSpec.MutualAuthentication)
+
 	return sdkObj
 }
 
@@ -327,6 +336,18 @@ func buildSDKCertificate(modelCert elbv2model.Certificate) *elbv2sdk.Certificate
 	}
 }
 
+// buildSDKMutualAuthenticationConfig builds the mutual TLS authentication config for listener
+func buildSDKMutualAuthenticationConfig(modelMutualAuthenticationCfg *elbv2model.MutualAuthenticationAttributes) *elbv2sdk.MutualAuthenticationAttributes {
+	if modelMutualAuthenticationCfg == nil {
+		return nil
+	}
+	return &elbv2sdk.MutualAuthenticationAttributes{
+		IgnoreClientCertificateExpiry: modelMutualAuthenticationCfg.IgnoreClientCertificateExpiry,
+		Mode:                          awssdk.String(modelMutualAuthenticationCfg.Mode),
+		TrustStoreArn:                 modelMutualAuthenticationCfg.TrustStoreArn,
+	}
+}
+
 func buildResListenerStatus(sdkLS ListenerWithTags) elbv2model.ListenerStatus {
 	return elbv2model.ListenerStatus{
 		ListenerARN: awssdk.StringValue(sdkLS.Listener.ListenerArn),

pkg/deploy/elbv2/listener_manager_test.go

@@ -10,10 +10,11 @@ import (
 
 func Test_isSDKListenerSettingsDrifted(t *testing.T) {
 	type args struct {
-		lsSpec                elbv2model.ListenerSpec
-		sdkLS                 ListenerWithTags
-		desiredDefaultActions []*elbv2sdk.Action
-		desiredDefaultCerts   []*elbv2sdk.Certificate
+		lsSpec                             elbv2model.ListenerSpec
+		sdkLS                              ListenerWithTags
+		desiredDefaultActions              []*elbv2sdk.Action
+		desiredDefaultCerts                []*elbv2sdk.Certificate
+		desiredDefaultMutualAuthentication *elbv2sdk.MutualAuthenticationAttributes
 	}
 	tests := []struct {
 		name string
@@ -49,6 +50,9 @@ func Test_isSDKListenerSettingsDrifted(t *testing.T) {
 						},
 						SslPolicy:  awssdk.String("ELBSecurityPolicy-FS-1-2-Res-2019-08"),
 						AlpnPolicy: awssdk.StringSlice([]string{"HTTP2Preferred"}),
+						MutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+							Mode: awssdk.String("off"),
+						},
 					},
 				},
 				desiredDefaultCerts: []*elbv2sdk.Certificate{
@@ -65,6 +69,9 @@ func Test_isSDKListenerSettingsDrifted(t *testing.T) {
 						},
 					},
 				},
+				desiredDefaultMutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+					Mode: awssdk.String("off"),
+				},
 			},
 		},
 		{
@@ -104,6 +111,9 @@ func Test_isSDKListenerSettingsDrifted(t *testing.T) {
 						},
 						SslPolicy:  awssdk.String("ELBSecurityPolicy-FS-1-2-Res-2019-08"),
 						AlpnPolicy: awssdk.StringSlice([]string{"HTTP2Preferred"}),
+						MutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+							Mode: awssdk.String("off"),
+						},
 					},
 				},
 				desiredDefaultCerts: []*elbv2sdk.Certificate{
@@ -120,6 +130,9 @@ func Test_isSDKListenerSettingsDrifted(t *testing.T) {
 						},
 					},
 				},
+				desiredDefaultMutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+					Mode: awssdk.String("off"),
+				},
 			},
 		},
 		{
@@ -154,6 +167,75 @@ func Test_isSDKListenerSettingsDrifted(t *testing.T) {
 						},
 						SslPolicy:  awssdk.String("ELBSecurityPolicy-FS-1-2-Res-2019-08"),
 						AlpnPolicy: awssdk.StringSlice([]string{"HTTP2Preferred"}),
+						MutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+							Mode: awssdk.String("off"),
+						},
+					},
+				},
+				desiredDefaultCerts: []*elbv2sdk.Certificate{
+					{
+						CertificateArn: awssdk.String("cert-arn1"),
+						IsDefault:      awssdk.Bool(true),
+					},
+				},
+				desiredDefaultActions: []*elbv2sdk.Action{
+					{
+						Type: awssdk.String("forward-config"),
+						ForwardConfig: &elbv2sdk.ForwardActionConfig{
+							TargetGroups: []*elbv2sdk.TargetGroupTuple{
+								{
+									TargetGroupArn: awssdk.String("target-group"),
+								},
+							},
+						},
+					},
+				},
+				desiredDefaultMutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+					Mode: awssdk.String("off"),
+				},
+			},
+		},
+		{
+			name: "listener hasn't drifted if mutualAuthentication verify mode specified",
+			args: args{
+				lsSpec: elbv2model.ListenerSpec{
+					Port:      80,
+					Protocol:  elbv2model.ProtocolHTTPS,
+					SSLPolicy: awssdk.String("ELBSecurityPolicy-FS-1-2-Res-2019-08"),
+					MutualAuthentication: &elbv2model.MutualAuthenticationAttributes{
+						Mode:          "verify",
+						TrustStoreArn: awssdk.String("arn:aws:elasticloadbalancing:us-east-1:123456789123:truststore/ts-1/8786hghf"),
+					},
+				},
+				sdkLS: ListenerWithTags{
+					Listener: &elbv2sdk.Listener{
+						Port:     awssdk.Int64(80),
+						Protocol: awssdk.String("HTTPS"),
+						Certificates: []*elbv2sdk.Certificate{
+							{
+								CertificateArn: awssdk.String("cert-arn1"),
+								IsDefault:      awssdk.Bool(true),
+							},
+						},
+						DefaultActions: []*elbv2sdk.Action{
+							{
+								Type: awssdk.String("forward-config"),
+								ForwardConfig: &elbv2sdk.ForwardActionConfig{
+									TargetGroups: []*elbv2sdk.TargetGroupTuple{
+										{
+											TargetGroupArn: awssdk.String("target-group"),
+										},
+									},
+								},
+							},
+						},
+						SslPolicy:  awssdk.String("ELBSecurityPolicy-FS-1-2-Res-2019-08"),
+						AlpnPolicy: awssdk.StringSlice([]string{"HTTP2Preferred"}),
+						MutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+							Mode:                          awssdk.String("verify"),
+							TrustStoreArn:                 awssdk.String("arn:aws:elasticloadbalancing:us-east-1:123456789123:truststore/ts-1/8786hghf"),
+							IgnoreClientCertificateExpiry: awssdk.Bool(false),
+						},
 					},
 				},
 				desiredDefaultCerts: []*elbv2sdk.Certificate{
@@ -174,12 +256,17 @@ func Test_isSDKListenerSettingsDrifted(t *testing.T) {
 						},
 					},
 				},
+				desiredDefaultMutualAuthentication: &elbv2sdk.MutualAuthenticationAttributes{
+					Mode:                          awssdk.String("verify"),
+					TrustStoreArn:                 awssdk.String("arn:aws:elasticloadbalancing:us-east-1:123456789123:truststore/ts-1/8786hghf"),
+					IgnoreClientCertificateExpiry: awssdk.Bool(false),
+				},
 			},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := isSDKListenerSettingsDrifted(tt.args.lsSpec, tt.args.sdkLS, tt.args.desiredDefaultActions, tt.args.desiredDefaultCerts)
+			got := isSDKListenerSettingsDrifted(tt.args.lsSpec, tt.args.sdkLS, tt.args.desiredDefaultActions, tt.args.desiredDefaultCerts, tt.args.desiredDefaultMutualAuthentication)
 			assert.Equal(t, tt.want, got)
 		})
 	}