Skip to content

Commit

Permalink
Add --default-load-balancer-scheme command line flag (#3908)
Browse files Browse the repository at this point in the history 
* ingress

* service

* service test syntax only

* ingress model_builder_test

* service model_builder_test

* refactor param name

* rename doc

* remove uneeded replace change

* rename tests
  • Loading branch information
@phuhung273
phuhung273 authored Dec 6, 2024
1 parent 283ebee commit 6a2dfee
Show file tree
Hide file tree
Showing 9 changed files with 377 additions and 88 deletions.
2 changes: 1 addition & 1 deletion controllers/ingress/group_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
annotationParser, subnetsResolver,
authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates,
cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, backendSGProvider, sgResolver,
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver,
controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger)
stackMarshaller := deploy.NewDefaultStackMarshaller()
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager,
Expand Down
2 changes: 1 addition & 1 deletion controllers/service/service_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde
serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger)
stackMarshaller := deploy.NewDefaultStackMarshaller()
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger)
Expand Down
1 change: 1 addition & 0 deletions docs/deploy/configurations.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne
| default-ssl-policy | string | ELBSecurityPolicy-2016-08 | Default SSL Policy that will be applied to all Ingresses or Services that do not have the SSL Policy annotation |
| default-tags | stringMap | | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority |
| default-target-type | string | instance | Default target type for Ingresses and Services - ip, instance |
| default-load-balancer-scheme | string | internal | Default scheme for ELBs - internal, internet-facing |
| [disable-ingress-class-annotation](#disable-ingress-class-annotation) | boolean | false | Disable new usage of the `kubernetes.io/ingress.class` annotation |
| [disable-ingress-group-name-annotation](#disable-ingress-group-name-annotation) | boolean | false | Disallow new use of the `alb.ingress.kubernetes.io/group.name` annotation |
| disable-restricted-sg-rules | boolean | false | Disable the usage of restricted security group rules |
Expand Down
18 changes: 18 additions & 0 deletions pkg/config/controller_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const (
flagK8sClusterName = "cluster-name"
flagDefaultTags = "default-tags"
flagDefaultTargetType = "default-target-type"
flagDefaultLoadBalancerScheme = "default-load-balancer-scheme"
flagExternalManagedTags = "external-managed-tags"
flagServiceTargetENISGTags = "service-target-eni-security-group-tags"
flagServiceMaxConcurrentReconciles = "service-max-concurrent-reconciles"
Expand Down Expand Up @@ -72,6 +73,9 @@ type ControllerConfig struct {
// Default target type for Ingress and Service objects
DefaultTargetType string

// Default scheme for ELB
DefaultLoadBalancerScheme string

// List of Tag keys on AWS resources that will be managed externally.
ExternalManagedTags []string

Expand Down Expand Up @@ -114,6 +118,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
"Default AWS Tags that will be applied to all AWS resources managed by this controller")
fs.StringVar(&cfg.DefaultTargetType, flagDefaultTargetType, string(elbv2.TargetTypeInstance),
"Default target type for Ingresses and Services - ip, instance")
fs.StringVar(&cfg.DefaultLoadBalancerScheme, flagDefaultLoadBalancerScheme, string(elbv2.LoadBalancerSchemeInternal),
"Default scheme for ELBs")
fs.StringSliceVar(&cfg.ExternalManagedTags, flagExternalManagedTags, nil,
"List of Tag keys on AWS resources that will be managed externally")
fs.IntVar(&cfg.ServiceMaxConcurrentReconciles, flagServiceMaxConcurrentReconciles, defaultMaxConcurrentReconciles,
Expand Down Expand Up @@ -162,6 +168,9 @@ func (cfg *ControllerConfig) Validate() error {
if err := cfg.validateDefaultTargetType(); err != nil {
return err
}
if err := cfg.validateDefaultLoadBalancerScheme(); err != nil {
return err
}
if err := cfg.validateBackendSecurityGroupConfiguration(); err != nil {
return err
}
Expand Down Expand Up @@ -205,6 +214,15 @@ func (cfg *ControllerConfig) validateDefaultTargetType() error {
}
}

func (cfg *ControllerConfig) validateDefaultLoadBalancerScheme() error {
switch cfg.DefaultLoadBalancerScheme {
case string(elbv2.LoadBalancerSchemeInternal), string(elbv2.LoadBalancerSchemeInternetFacing):
return nil
default:
return errors.Errorf("invalid value %v for default scheme", cfg.DefaultLoadBalancerScheme)
}
}

func (cfg *ControllerConfig) validateBackendSecurityGroupConfiguration() error {
if len(cfg.BackendSecurityGroup) == 0 {
return nil
Expand Down
92 changes: 47 additions & 45 deletions pkg/ingress/model_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,37 +42,38 @@ func NewDefaultModelBuilder(k8sClient client.Client, eventRecorder record.EventR
annotationParser annotations.Parser, subnetsResolver networkingpkg.SubnetsResolver,
authConfigBuilder AuthConfigBuilder, enhancedBackendBuilder EnhancedBackendBuilder,
trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, featureGates config.FeatureGates,
vpcID string, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string,
vpcID string, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string,
backendSGProvider networkingpkg.BackendSGProvider, sgResolver networkingpkg.SecurityGroupResolver,
enableBackendSG bool, disableRestrictedSGRules bool, allowedCAARNs []string, enableIPTargetType bool, logger logr.Logger) *defaultModelBuilder {
certDiscovery := NewACMCertDiscovery(acmClient, allowedCAARNs, logger)
ruleOptimizer := NewDefaultRuleOptimizer(logger)
return &defaultModelBuilder{
k8sClient: k8sClient,
eventRecorder: eventRecorder,
ec2Client: ec2Client,
elbv2Client: elbv2Client,
vpcID: vpcID,
clusterName: clusterName,
annotationParser: annotationParser,
subnetsResolver: subnetsResolver,
backendSGProvider: backendSGProvider,
sgResolver: sgResolver,
certDiscovery: certDiscovery,
authConfigBuilder: authConfigBuilder,
enhancedBackendBuilder: enhancedBackendBuilder,
ruleOptimizer: ruleOptimizer,
trackingProvider: trackingProvider,
elbv2TaggingManager: elbv2TaggingManager,
featureGates: featureGates,
defaultTags: defaultTags,
externalManagedTags: sets.NewString(externalManagedTags...),
defaultSSLPolicy: defaultSSLPolicy,
defaultTargetType: elbv2model.TargetType(defaultTargetType),
enableBackendSG: enableBackendSG,
disableRestrictedSGRules: disableRestrictedSGRules,
enableIPTargetType: enableIPTargetType,
logger: logger,
k8sClient: k8sClient,
eventRecorder: eventRecorder,
ec2Client: ec2Client,
elbv2Client: elbv2Client,
vpcID: vpcID,
clusterName: clusterName,
annotationParser: annotationParser,
subnetsResolver: subnetsResolver,
backendSGProvider: backendSGProvider,
sgResolver: sgResolver,
certDiscovery: certDiscovery,
authConfigBuilder: authConfigBuilder,
enhancedBackendBuilder: enhancedBackendBuilder,
ruleOptimizer: ruleOptimizer,
trackingProvider: trackingProvider,
elbv2TaggingManager: elbv2TaggingManager,
featureGates: featureGates,
defaultTags: defaultTags,
externalManagedTags: sets.NewString(externalManagedTags...),
defaultSSLPolicy: defaultSSLPolicy,
defaultTargetType: elbv2model.TargetType(defaultTargetType),
defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme),
enableBackendSG: enableBackendSG,
disableRestrictedSGRules: disableRestrictedSGRules,
enableIPTargetType: enableIPTargetType,
logger: logger,
}
}

Expand All @@ -88,24 +89,25 @@ type defaultModelBuilder struct {
vpcID string
clusterName string

annotationParser annotations.Parser
subnetsResolver networkingpkg.SubnetsResolver
backendSGProvider networkingpkg.BackendSGProvider
sgResolver networkingpkg.SecurityGroupResolver
certDiscovery CertDiscovery
authConfigBuilder AuthConfigBuilder
enhancedBackendBuilder EnhancedBackendBuilder
ruleOptimizer RuleOptimizer
trackingProvider tracking.Provider
elbv2TaggingManager elbv2deploy.TaggingManager
featureGates config.FeatureGates
defaultTags map[string]string
externalManagedTags sets.String
defaultSSLPolicy string
defaultTargetType elbv2model.TargetType
enableBackendSG bool
disableRestrictedSGRules bool
enableIPTargetType bool
annotationParser annotations.Parser
subnetsResolver networkingpkg.SubnetsResolver
backendSGProvider networkingpkg.BackendSGProvider
sgResolver networkingpkg.SecurityGroupResolver
certDiscovery CertDiscovery
authConfigBuilder AuthConfigBuilder
enhancedBackendBuilder EnhancedBackendBuilder
ruleOptimizer RuleOptimizer
trackingProvider tracking.Provider
elbv2TaggingManager elbv2deploy.TaggingManager
featureGates config.FeatureGates
defaultTags map[string]string
externalManagedTags sets.String
defaultSSLPolicy string
defaultTargetType elbv2model.TargetType
defaultLoadBalancerScheme elbv2model.LoadBalancerScheme
enableBackendSG bool
disableRestrictedSGRules bool
enableIPTargetType bool

logger logr.Logger
}
Expand Down Expand Up @@ -142,7 +144,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group) (core.S
defaultTags: b.defaultTags,
externalManagedTags: b.externalManagedTags,
defaultIPAddressType: elbv2model.IPAddressTypeIPV4,
defaultScheme: elbv2model.LoadBalancerSchemeInternal,
defaultScheme: b.defaultLoadBalancerScheme,
defaultSSLPolicy: b.defaultSSLPolicy,
defaultTargetType: b.defaultTargetType,
defaultBackendProtocol: elbv2model.ProtocolHTTP,
Expand Down
128 changes: 118 additions & 10 deletions pkg/ingress/model_builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -605,14 +605,15 @@ func Test_defaultModelBuilder_Build(t *testing.T) {
}

tests := []struct {
name string
env env
defaultTargetType string
enableIPTargetType *bool
args args
fields fields
wantStackPatch string
wantErr string
name string
env env
defaultTargetType string
defaultLoadBalancerScheme string
enableIPTargetType *bool
args args
fields fields
wantStackPatch string
wantErr string
}{
{
name: "Ingress - vanilla internal",
Expand Down Expand Up @@ -3628,6 +3629,108 @@ func Test_defaultModelBuilder_Build(t *testing.T) {
}
}
}
}`,
},
{
name: "Ingress - vanilla with default-load-balancer-scheme internet-facing",
env: env{
svcs: []*corev1.Service{ns_1_svc_1, ns_1_svc_2, ns_1_svc_3},
},
fields: fields{
resolveViaDiscoveryCalls: []resolveViaDiscoveryCall{resolveViaDiscoveryCallForInternetFacingLB},
listLoadBalancersCalls: []listLoadBalancersCall{listLoadBalancerCallForEmptyLB},
enableBackendSG: true,
},
defaultLoadBalancerScheme: string(elbv2model.LoadBalancerSchemeInternetFacing),
args: args{
ingGroup: Group{
ID: GroupID{Namespace: "ns-1", Name: "ing-1"},
Members: []ClassifiedIngress{
{
Ing: &networking.Ingress{ObjectMeta: metav1.ObjectMeta{
Namespace: "ns-1",
Name: "ing-1",
},
Spec: networking.IngressSpec{
Rules: []networking.IngressRule{
{
Host: "app-1.example.com",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/svc-1",
Backend: networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: ns_1_svc_1.Name,
Port: networking.ServiceBackendPort{
Name: "http",
},
},
},
},
{
Path: "/svc-2",
Backend: networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: ns_1_svc_2.Name,
Port: networking.ServiceBackendPort{
Name: "http",
},
},
},
},
},
},
},
},
{
Host: "app-2.example.com",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/svc-3",
Backend: networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: ns_1_svc_3.Name,
Port: networking.ServiceBackendPort{
Name: "https",
},
},
},
},
},
},
},
},
},
},
},
},
},
},
},
wantStackPatch: `
{
"resources": {
"AWS::ElasticLoadBalancingV2::LoadBalancer": {
"LoadBalancer": {
"spec": {
"name": "k8s-ns1-ing1-159dd7a143",
"scheme": "internet-facing",
"subnetMapping": [
{
"subnetID": "subnet-c"
},
{
"subnetID": "subnet-d"
}
]
}
}
}
}
}`,
},
}
Expand Down Expand Up @@ -3681,6 +3784,10 @@ func Test_defaultModelBuilder_Build(t *testing.T) {
if defaultTargetType == "" {
defaultTargetType = "instance"
}
defaultLoadBalancerScheme := tt.defaultLoadBalancerScheme
if defaultLoadBalancerScheme == "" {
defaultLoadBalancerScheme = string(elbv2model.LoadBalancerSchemeInternal)
}

b := &defaultModelBuilder{
k8sClient: k8sClient,
Expand All @@ -3703,8 +3810,9 @@ func Test_defaultModelBuilder_Build(t *testing.T) {
featureGates: config.NewFeatureGates(),
logger: logr.New(&log.NullLogSink{}),

defaultSSLPolicy: "ELBSecurityPolicy-2016-08",
defaultTargetType: elbv2model.TargetType(defaultTargetType),
defaultSSLPolicy: "ELBSecurityPolicy-2016-08",
defaultTargetType: elbv2model.TargetType(defaultTargetType),
defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme),
}

if tt.enableIPTargetType == nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/service/model_build_load_balancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerScheme(ctx context.Context) (el
return "", errors.New("invalid load balancer scheme")
}
}
return elbv2model.LoadBalancerSchemeInternal, nil
return t.defaultLoadBalancerScheme, nil
}

func (t *defaultModelBuildTask) buildLoadBalancerSchemeViaAnnotation(ctx context.Context) (elbv2model.LoadBalancerScheme, bool, error) {
Expand Down
Loading

0 comments on commit 6a2dfee

Please sign in to comment.