From 5ce1f8a76d9fac92becf520b62ccb7040c33cab7 Mon Sep 17 00:00:00 2001 From: Michael Shen Date: Sat, 10 Dec 2022 23:16:40 -0500 Subject: [PATCH] Add validation for default ingresscontroller load balancer Signed-off-by: Michael Shen --- go.mod | 1 + go.sum | 2 + main.go | 1 + pkg/mirrosa/api_loadbalancer.go | 12 +- pkg/mirrosa/ingresscontroller_loadbalancer.go | 164 ++++++++++++++++++ 5 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 pkg/mirrosa/ingresscontroller_loadbalancer.go diff --git a/go.mod b/go.mod index c5e2685..926162c 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.8 github.com/aws/aws-sdk-go-v2/credentials v1.13.8 github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.0 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.0 github.com/aws/aws-sdk-go-v2/service/route53 v1.26.0 github.com/openshift-online/ocm-cli v0.1.65 diff --git a/go.sum b/go.sum index 554755e..fc4437f 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk= github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.0 h1:FFfQypN9iItIrGhbl8em90uXMFBLrCkNC1yJ65+m9Sk= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.0/go.mod h1:3OUv9SlYvymsCF3I5NftITc1+B09cF5lg4IsZgQQy1U= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.0 h1:Fs+mQ2VSOH3YhNJcfImnl7dsKAm/gqw4Q9iqLRIiPWE= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.0/go.mod h1:ix71C17la8K2MUJrqJzu+i7+aPoQYTAy14hKQbGDB9w= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= diff --git a/main.go b/main.go index 191b0d0..4ad5276 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ func main() { sugared.Infof("%s: \"Mirror mirror on the wall, who's the fairest of them all?\"", mirrosa.ClusterInfo.Name) if err := mirrosa.ValidateComponents(context.TODO(), + mirrosa.NewIngressControllerLoadBalancer(), mirrosa.NewVpc(), mirrosa.NewDhcpOptions(), mirrosa.NewSecurityGroup(), diff --git a/pkg/mirrosa/api_loadbalancer.go b/pkg/mirrosa/api_loadbalancer.go index 0bdee7a..00f0df2 100644 --- a/pkg/mirrosa/api_loadbalancer.go +++ b/pkg/mirrosa/api_loadbalancer.go @@ -20,8 +20,8 @@ const ( "\n - An internal (-int) NLB to balance traffic within the cluster's VPC." ) -// elb represents the expected state of an Elastic Load Balancer in AWS -type elb struct { +// mirrosaElb represents the expected state of an Elastic Load Balancer in AWS +type mirrosaElb struct { name string expectedListeners map[string]listener } @@ -143,10 +143,10 @@ func (n NetworkLoadBalancer) FilterValue() string { } // getExpectedNLBs returns a map of expected elb instances given a NetworkLoadBalancer Component -func (n NetworkLoadBalancer) getExpectedNLBs() map[string]elb { - expected := map[string]elb{} +func (n NetworkLoadBalancer) getExpectedNLBs() map[string]mirrosaElb { + expected := map[string]mirrosaElb{} - expected["api-int"] = elb{ + expected["api-int"] = mirrosaElb{ name: fmt.Sprintf("%s-int", n.InfraName), expectedListeners: map[string]listener{ "etcd": { @@ -164,7 +164,7 @@ func (n NetworkLoadBalancer) getExpectedNLBs() map[string]elb { // TODO: Handle non-STS, where the external NLB is optional if it is a private cluster if !n.PrivateLink && n.Sts { - expected["api-ext"] = elb{ + expected["api-ext"] = mirrosaElb{ name: fmt.Sprintf("%s-ext", n.InfraName), expectedListeners: map[string]listener{ "kube-apiserver": { diff --git a/pkg/mirrosa/ingresscontroller_loadbalancer.go b/pkg/mirrosa/ingresscontroller_loadbalancer.go new file mode 100644 index 0000000..ad3db0e --- /dev/null +++ b/pkg/mirrosa/ingresscontroller_loadbalancer.go @@ -0,0 +1,164 @@ +package mirrosa + +import ( + "context" + "errors" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + elb "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" + "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing/types" + elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + "go.uber.org/zap" +) + +const ( + defaultIngressControllerLoadBalancerTagKey = "kubernetes.io/service-name" + defaultIngressControllerLoadBalancerTagValue = "openshift-ingress/router-default" + defaultIngressControllerLoadBalancerDescription = "TODO" +) + +var _ Component = &DefaultIngressControllerLoadBalancer{} + +type DefaultIngressControllerLoadBalancer struct { + log *zap.SugaredLogger + InfraName string + + Ec2Client Ec2AwsApi + ElbClient *elb.Client + ElbV2Client NetworkLoadBalancerAPIClient +} + +func (c *Client) NewIngressControllerLoadBalancer() DefaultIngressControllerLoadBalancer { + return DefaultIngressControllerLoadBalancer{ + log: c.log, + InfraName: c.ClusterInfo.InfraName, + Ec2Client: ec2.NewFromConfig(c.AwsConfig), + ElbClient: elb.NewFromConfig(c.AwsConfig), + ElbV2Client: elbv2.NewFromConfig(c.AwsConfig), + } +} + +func (d DefaultIngressControllerLoadBalancer) Validate(ctx context.Context) error { + d.log.Info("searching for default ingress controller load balancer") + + d.log.Info("searching classic load balancers") + clb, err := d.searchForCLB(ctx) + if err != nil { + d.log.Info("failed to find classic load balancer for default ingress controller") + // TODO: Search NLBs + } + + if len(clb.SecurityGroups) != 1 { + return fmt.Errorf("expected 1 security group attached to the default ingress controller load balancer, found %d", len(clb.SecurityGroups)) + } + + resp, err := d.Ec2Client.DescribeSecurityGroupRules(ctx, &ec2.DescribeSecurityGroupRulesInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String("group-id"), + Values: []string{clb.SecurityGroups[0]}, + }, + }, + }) + if err != nil { + return err + } + + expectedRules := map[string]securityGroupRule{ + "http": { + CidrIpv4: "0.0.0.0/0", + IpProtocol: ec2Types.ProtocolTcp, + FromPort: 80, + ToPort: 80, + IsEgress: false, + }, + "https": { + CidrIpv4: "0.0.0.0/0", + IpProtocol: ec2Types.ProtocolTcp, + FromPort: 443, + ToPort: 443, + IsEgress: false, + }, + "icmp": { + CidrIpv4: "0.0.0.0/0", + IpProtocol: "icmp", + FromPort: 3, + ToPort: 4, + IsEgress: false, + }, + "egress": { + CidrIpv4: "0.0.0.0/0", + IpProtocol: "-1", + FromPort: -1, + ToPort: -1, + IsEgress: true, + }, + } + + for _, rule := range resp.SecurityGroupRules { + // If we've found all the required security group rules, stop + if len(expectedRules) == 0 { + break + } + + d.log.Debugf("found security group rule %s", *rule.SecurityGroupRuleId) + for k, expectedRule := range expectedRules { + if compareSecurityGroupRules(expectedRule, rule) { + d.log.Infof("security group rule validated for %s: %+v", k, expectedRule) + delete(expectedRules, k) + } + } + } + + if len(expectedRules) > 0 { + return fmt.Errorf("missing required rules in default ingress controller load balancer security group %v", expectedRules) + } + + return nil +} + +func (d DefaultIngressControllerLoadBalancer) Documentation() string { + return defaultIngressControllerLoadBalancerDescription +} + +func (d DefaultIngressControllerLoadBalancer) FilterValue() string { + return "Default Ingress Controller Load Balancer" +} + +func (d DefaultIngressControllerLoadBalancer) searchForCLB(ctx context.Context) (*types.LoadBalancerDescription, error) { + resp, err := d.ElbClient.DescribeLoadBalancers(ctx, &elb.DescribeLoadBalancersInput{}) + if err != nil { + return nil, fmt.Errorf("failed to describe CLBs: %w", err) + } + + for _, clb := range resp.LoadBalancerDescriptions { + expectedTags := map[string]string{ + defaultIngressControllerLoadBalancerTagKey: defaultIngressControllerLoadBalancerTagValue, + fmt.Sprintf("kubernetes.io/cluster/%s", d.InfraName): "owned", + } + + tagsResp, err := d.ElbClient.DescribeTags(ctx, &elb.DescribeTagsInput{ + LoadBalancerNames: []string{*clb.LoadBalancerName}, + }) + if err != nil { + return nil, fmt.Errorf("failed to describe tags of CLB %s:%w", *clb.LoadBalancerName, err) + } + + for _, tag := range tagsResp.TagDescriptions[0].Tags { + if len(expectedTags) == 0 { + return &clb, nil + } + + if v, ok := expectedTags[*tag.Key]; ok { + if v == *tag.Value { + d.log.Infof("found match for tag %s:%s", *tag.Key, *tag.Value) + delete(expectedTags, *tag.Key) + } + } + } + } + + return nil, errors.New("no matching CLB found") +}