From 0722acb8e812890461663fad39bb834a56326818 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 15 May 2024 17:35:36 +0200 Subject: [PATCH] Upgrade to aws-load-balancer-controller for NLB Services --- Makefile | 2 +- api/v1alpha1/backend_types.go | 1 + api/v1alpha1/common_types.go | 12 +- api/v1alpha1/echoapi_types.go | 1 + api/v1alpha1/zz_generated.deepcopy.go | 10 ++ .../saas-operator.clusterserviceversion.yaml | 20 ++- .../manifests/saas.3scale.net_backends.yaml | 6 + .../manifests/saas.3scale.net_echoapis.yaml | 6 + .../crd/bases/saas.3scale.net_backends.yaml | 6 + .../crd/bases/saas.3scale.net_echoapis.yaml | 6 + config/manager/kustomization.yaml | 2 +- .../saas-operator.clusterserviceversion.yaml | 12 ++ controllers/backend_controller.go | 38 +++- controllers/backend_controller_suite_test.go | 31 ++-- controllers/echoapi_controller.go | 39 ++++- controllers/echoapi_controller_suite_test.go | 4 +- controllers/upgrade_controller.go | 165 ++++++++++++++++++ docs/api-reference/reference.asciidoc | 41 ++++- pkg/generators/backend/services.go | 2 +- pkg/generators/echoapi/service.go | 2 +- pkg/resource_builders/service/util.go | 25 ++- pkg/util/k8s.go | 14 ++ pkg/version/version.go | 2 +- 23 files changed, 408 insertions(+), 39 deletions(-) create mode 100644 controllers/upgrade_controller.go create mode 100644 pkg/util/k8s.go diff --git a/Makefile b/Makefile index d86cb3f5..8c1cdf51 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.23.0-alpha.9 +VERSION ?= 0.24.0-alpha.1 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") # To re-generate a bundle for other specific channels without changing the standard setup, you can: diff --git a/api/v1alpha1/backend_types.go b/api/v1alpha1/backend_types.go index aa095ce1..cd696354 100644 --- a/api/v1alpha1/backend_types.go +++ b/api/v1alpha1/backend_types.go @@ -48,6 +48,7 @@ var ( backendDefaultListenerNLBLoadBalancer defaultNLBLoadBalancerSpec = defaultNLBLoadBalancerSpec{ ProxyProtocol: util.Pointer(true), CrossZoneLoadBalancingEnabled: util.Pointer(true), + TerminationProtection: util.Pointer(false), } backendDefaultListenerReplicas int32 = 2 backendDefaultListenerResources defaultResourceRequirementsSpec = defaultResourceRequirementsSpec{ diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index a2093282..aa340edb 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -247,17 +247,25 @@ type NLBLoadBalancerSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional EIPAllocations []string `json:"eipAllocations,omitempty"` + // Optionally specify the load balancer name + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + LoadBalancerName *string `json:"loadBalancerName,omitempty"` + // Termination protection setting + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + TerminationProtection *bool `json:"terminationProtection,omitempty"` } type defaultNLBLoadBalancerSpec struct { - CrossZoneLoadBalancingEnabled, ProxyProtocol *bool - EIPAllocations []string + CrossZoneLoadBalancingEnabled, ProxyProtocol, TerminationProtection *bool } // Default sets default values for any value not specifically set in the NLBLoadBalancerSpec struct func (spec *NLBLoadBalancerSpec) Default(def defaultNLBLoadBalancerSpec) { spec.ProxyProtocol = boolOrDefault(spec.ProxyProtocol, def.ProxyProtocol) spec.CrossZoneLoadBalancingEnabled = boolOrDefault(spec.CrossZoneLoadBalancingEnabled, def.CrossZoneLoadBalancingEnabled) + spec.TerminationProtection = boolOrDefault(spec.TerminationProtection, def.TerminationProtection) } // IsDeactivated true if the field is set with the deactivated value (empty struct) diff --git a/api/v1alpha1/echoapi_types.go b/api/v1alpha1/echoapi_types.go index 74b6916c..0b562e4c 100644 --- a/api/v1alpha1/echoapi_types.go +++ b/api/v1alpha1/echoapi_types.go @@ -68,6 +68,7 @@ var ( echoapiDefaultNLBLoadBalancer defaultNLBLoadBalancerSpec = defaultNLBLoadBalancerSpec{ ProxyProtocol: util.Pointer(true), CrossZoneLoadBalancingEnabled: util.Pointer(true), + TerminationProtection: util.Pointer(false), } ) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 5cbf08c4..b50f39eb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1958,6 +1958,16 @@ func (in *NLBLoadBalancerSpec) DeepCopyInto(out *NLBLoadBalancerSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.LoadBalancerName != nil { + in, out := &in.LoadBalancerName, &out.LoadBalancerName + *out = new(string) + **out = **in + } + if in.TerminationProtection != nil { + in, out := &in.TerminationProtection, &out.TerminationProtection + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NLBLoadBalancerSpec. diff --git a/bundle/manifests/saas-operator.clusterserviceversion.yaml b/bundle/manifests/saas-operator.clusterserviceversion.yaml index 741f8648..cfd4665e 100644 --- a/bundle/manifests/saas-operator.clusterserviceversion.yaml +++ b/bundle/manifests/saas-operator.clusterserviceversion.yaml @@ -598,7 +598,7 @@ metadata: capabilities: Basic Install categories: Integration & Delivery containerImage: quay.io/3scale/saas-operator - createdAt: "2024-05-06T14:15:35Z" + createdAt: "2024-05-15T15:32:45Z" description: |- The 3scale SaaS Operator creates and maintains a SaaS-ready deployment of the Red Hat 3scale API Management on OpenShift. @@ -606,7 +606,7 @@ metadata: operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 repository: https://github.com/3scale-ops/saas-operator support: Red Hat - name: saas-operator.v0.23.0-alpha.9 + name: saas-operator.v0.24.0-alpha.1 namespace: placeholder spec: apiservicedefinitions: {} @@ -1602,9 +1602,15 @@ spec: - description: The list of optional Elastic IPs allocations displayName: EIPAllocations path: listener.loadBalancer.eipAllocations + - description: Optionally specify the load balancer name + displayName: Load Balancer Name + path: listener.loadBalancer.loadBalancerName - description: Enables/disbles use of proxy protocol in the load balancer displayName: Proxy Protocol path: listener.loadBalancer.proxyProtocol + - description: Termination protection setting + displayName: Termination Protection + path: listener.loadBalancer.terminationProtection - description: Marin3r configures the Marin3r sidecars for the component displayName: Marin3r path: listener.marin3r @@ -2179,9 +2185,15 @@ spec: - description: The list of optional Elastic IPs allocations displayName: EIPAllocations path: loadBalancer.eipAllocations + - description: Optionally specify the load balancer name + displayName: Load Balancer Name + path: loadBalancer.loadBalancerName - description: Enables/disbles use of proxy protocol in the load balancer displayName: Proxy Protocol path: loadBalancer.proxyProtocol + - description: Termination protection setting + displayName: Termination Protection + path: loadBalancer.terminationProtection - description: Marin3r configures the Marin3r sidecars for the component displayName: Marin3r path: marin3r @@ -4610,7 +4622,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] - image: quay.io/3scale/saas-operator:v0.23.0-alpha.9 + image: quay.io/3scale/saas-operator:v0.24.0-alpha.1 livenessProbe: httpGet: path: /healthz @@ -5174,4 +5186,4 @@ spec: provider: name: Red Hat url: https://www.3scale.net/ - version: 0.23.0-alpha.9 + version: 0.24.0-alpha.1 diff --git a/bundle/manifests/saas.3scale.net_backends.yaml b/bundle/manifests/saas.3scale.net_backends.yaml index 30bd2bab..61d3bfed 100644 --- a/bundle/manifests/saas.3scale.net_backends.yaml +++ b/bundle/manifests/saas.3scale.net_backends.yaml @@ -827,10 +827,16 @@ spec: items: type: string type: array + loadBalancerName: + description: Optionally specify the load balancer name + type: string proxyProtocol: description: Enables/disbles use of proxy protocol in the load balancer type: boolean + terminationProtection: + description: Termination protection setting + type: boolean type: object marin3r: description: Marin3r configures the Marin3r sidecars for the component diff --git a/bundle/manifests/saas.3scale.net_echoapis.yaml b/bundle/manifests/saas.3scale.net_echoapis.yaml index f5b7c6f6..7eeca343 100644 --- a/bundle/manifests/saas.3scale.net_echoapis.yaml +++ b/bundle/manifests/saas.3scale.net_echoapis.yaml @@ -248,10 +248,16 @@ spec: items: type: string type: array + loadBalancerName: + description: Optionally specify the load balancer name + type: string proxyProtocol: description: Enables/disbles use of proxy protocol in the load balancer type: boolean + terminationProtection: + description: Termination protection setting + type: boolean type: object marin3r: description: Marin3r configures the Marin3r sidecars for the component diff --git a/config/crd/bases/saas.3scale.net_backends.yaml b/config/crd/bases/saas.3scale.net_backends.yaml index 5aee1f62..fcd7a602 100644 --- a/config/crd/bases/saas.3scale.net_backends.yaml +++ b/config/crd/bases/saas.3scale.net_backends.yaml @@ -828,10 +828,16 @@ spec: items: type: string type: array + loadBalancerName: + description: Optionally specify the load balancer name + type: string proxyProtocol: description: Enables/disbles use of proxy protocol in the load balancer type: boolean + terminationProtection: + description: Termination protection setting + type: boolean type: object marin3r: description: Marin3r configures the Marin3r sidecars for the component diff --git a/config/crd/bases/saas.3scale.net_echoapis.yaml b/config/crd/bases/saas.3scale.net_echoapis.yaml index 662bf4cb..8d2922f0 100644 --- a/config/crd/bases/saas.3scale.net_echoapis.yaml +++ b/config/crd/bases/saas.3scale.net_echoapis.yaml @@ -249,10 +249,16 @@ spec: items: type: string type: array + loadBalancerName: + description: Optionally specify the load balancer name + type: string proxyProtocol: description: Enables/disbles use of proxy protocol in the load balancer type: boolean + terminationProtection: + description: Termination protection setting + type: boolean type: object marin3r: description: Marin3r configures the Marin3r sidecars for the component diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index e28b324e..8cb83796 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,4 +13,4 @@ kind: Kustomization images: - name: controller newName: quay.io/3scale/saas-operator - newTag: v0.23.0-alpha.9 + newTag: v0.24.0-alpha.1 diff --git a/config/manifests/bases/saas-operator.clusterserviceversion.yaml b/config/manifests/bases/saas-operator.clusterserviceversion.yaml index 2f1aadbd..154807a9 100644 --- a/config/manifests/bases/saas-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/saas-operator.clusterserviceversion.yaml @@ -1143,9 +1143,15 @@ spec: - description: The list of optional Elastic IPs allocations displayName: EIPAllocations path: listener.loadBalancer.eipAllocations + - description: Optionally specify the load balancer name + displayName: Load Balancer Name + path: listener.loadBalancer.loadBalancerName - description: Enables/disbles use of proxy protocol in the load balancer displayName: Proxy Protocol path: listener.loadBalancer.proxyProtocol + - description: Termination protection setting + displayName: Termination Protection + path: listener.loadBalancer.terminationProtection - description: Marin3r configures the Marin3r sidecars for the component displayName: Marin3r path: listener.marin3r @@ -1720,9 +1726,15 @@ spec: - description: The list of optional Elastic IPs allocations displayName: EIPAllocations path: loadBalancer.eipAllocations + - description: Optionally specify the load balancer name + displayName: Load Balancer Name + path: loadBalancer.loadBalancerName - description: Enables/disbles use of proxy protocol in the load balancer displayName: Proxy Protocol path: loadBalancer.proxyProtocol + - description: Termination protection setting + displayName: Termination Protection + path: loadBalancer.terminationProtection - description: Marin3r configures the Marin3r sidecars for the component displayName: Marin3r path: marin3r diff --git a/controllers/backend_controller.go b/controllers/backend_controller.go index 0f21321f..bc1b2cc9 100644 --- a/controllers/backend_controller.go +++ b/controllers/backend_controller.go @@ -18,13 +18,17 @@ package controllers import ( "context" + "strings" "github.com/3scale-ops/basereconciler/reconciler" "github.com/3scale-ops/basereconciler/util" saasv1alpha1 "github.com/3scale-ops/saas-operator/api/v1alpha1" "github.com/3scale-ops/saas-operator/pkg/generators/backend" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) // BackendReconciler reconciles a Backend object @@ -49,7 +53,7 @@ type BackendReconciler struct { // move the current state of the cluster closer to the desired state. func (r *BackendReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - ctx, _ = r.Logger(ctx, "name", req.Name, "namespace", req.Namespace) + ctx, logger := r.Logger(ctx, "name", req.Name, "namespace", req.Namespace) instance := &saasv1alpha1.Backend{} result := r.ManageResourceLifecycle(ctx, req, instance, reconciler.WithInMemoryInitializationFunc(util.ResourceDefaulter(instance))) @@ -61,6 +65,38 @@ func (r *BackendReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if err != nil { return ctrl.Result{}, err } + + // Upgrade NLBs managed by nlb-helper-operator + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: gen.Listener.GetComponent(), Namespace: req.Namespace}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err == nil { + // 1. Update parent resource with NLB name for NLB adoption + // 2. Add termination protection + nlbName := strings.Split(strings.Split(svc.Status.LoadBalancer.Ingress[0].Hostname, ".")[0], "-")[0] + if instance.Spec.Listener.LoadBalancer.LoadBalancerName == nil || + *instance.Spec.Listener.LoadBalancer.LoadBalancerName != nlbName { + patch := client.MergeFrom(instance.DeepCopy()) + instance.Spec.Listener.LoadBalancer.LoadBalancerName = &nlbName + instance.Spec.Listener.LoadBalancer.TerminationProtection = util.Pointer(true) + if err := r.Client.Patch(ctx, instance, patch); err != nil { + return ctrl.Result{}, err + } + logger.Info("resource patched", "kind", "Backend", "key", req) + } + + // 3. Abandon old Service resource + if svc.GetOwnerReferences() != nil || svc.GetAnnotations()["service.beta.kubernetes.io/aws-load-balancer-type"] != "external" { + svc.ObjectMeta.OwnerReferences = nil + svc.ObjectMeta.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"] = "external" + if err := r.Client.Update(ctx, svc); err != nil { + return ctrl.Result{}, err + } + logger.Info("resource abandoned", "kind", "Service", "key", req) + } + + } else if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + resources, err := gen.Resources() if err != nil { return ctrl.Result{}, err diff --git a/controllers/backend_controller_suite_test.go b/controllers/backend_controller_suite_test.go index 1e61caf5..06510878 100644 --- a/controllers/backend_controller_suite_test.go +++ b/controllers/backend_controller_suite_test.go @@ -142,9 +142,9 @@ var _ = Describe("Backend controller", func() { }).Assert(k8sClient, dep, timeout, poll)) svc := &corev1.Service{} - By("deploying the backend-listener service", + By("deploying the backend-listener-nlb service", (&testutil.ExpectedResource{ - Name: "backend-listener", Namespace: namespace, + Name: "backend-listener-nlb", Namespace: namespace, }).Assert(k8sClient, svc, timeout, poll)) Expect(svc.Spec.Selector["deployment"]).To(Equal("backend-listener")) @@ -290,8 +290,8 @@ var _ = Describe("Backend controller", func() { return err } - rvs["svc/backend-listener"] = testutil.GetResourceVersion( - k8sClient, &corev1.Service{}, "backend-listener", namespace, timeout, poll) + rvs["svc/backend-listener-nlb"] = testutil.GetResourceVersion( + k8sClient, &corev1.Service{}, "backend-listener-nlb", namespace, timeout, poll) rvs["deployment/backend-listener"] = testutil.GetResourceVersion( k8sClient, &appsv1.Deployment{}, "backend-listener", namespace, timeout, poll) rvs["hpa/backend-worker"] = testutil.GetResourceVersion( @@ -376,14 +376,15 @@ var _ = Describe("Backend controller", func() { }).Assert(k8sClient, dep, timeout, poll)) svc := &corev1.Service{} - By("updating backend-listener service", + By("updating backend-listener-nlb service", (&testutil.ExpectedResource{ - Name: "backend-listener", Namespace: namespace, - LastVersion: rvs["svc/backend-listener"], + Name: "backend-listener-nlb", Namespace: namespace, + LastVersion: rvs["svc/backend-listener-nlb"], }).Assert(k8sClient, svc, timeout, poll)) Expect(svc.Spec.Selector["deployment"]).To(Equal("backend-listener")) - Expect(svc.GetAnnotations()["service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled"]).To(Equal("false")) + Expect(svc.GetAnnotations()["service.beta.kubernetes.io/aws-load-balancer-attributes"]). + To(Equal("load_balancing.cross_zone.enabled=false,deletion_protection.enabled=false")) hpa := &autoscalingv2.HorizontalPodAutoscaler{} By("updating the backend-worker workload", @@ -482,7 +483,7 @@ var _ = Describe("Backend controller", func() { } rvs["svc/backend-listener"] = testutil.GetResourceVersion( - k8sClient, &corev1.Service{}, "backend-listener", namespace, timeout, poll) + k8sClient, &corev1.Service{}, "backend-listener-nlb", namespace, timeout, poll) rvs["deployment/backend-listener"] = testutil.GetResourceVersion( k8sClient, &appsv1.Deployment{}, "backend-listener", namespace, timeout, poll) rvs["deployment/backend-worker"] = testutil.GetResourceVersion( @@ -541,9 +542,9 @@ var _ = Describe("Backend controller", func() { }).Assert(k8sClient, dep, timeout, poll)) svc := &corev1.Service{} - By("keeps the backend-listener service deployment label selector", + By("keeps the backend-listener-nlb service deployment label selector", (&testutil.ExpectedResource{ - Name: "backend-listener", Namespace: namespace, + Name: "backend-listener-nlb", Namespace: namespace, }).Assert(k8sClient, svc, timeout, poll)) Expect(svc.Spec.Selector["deployment"]).To(Equal("backend-listener")) @@ -594,8 +595,8 @@ var _ = Describe("Backend controller", func() { rvs["deployment/backend-listener-canary"] = testutil.GetResourceVersion( k8sClient, &appsv1.Deployment{}, "backend-listener-canary", namespace, timeout, poll) - rvs["svc/backend-listener"] = testutil.GetResourceVersion( - k8sClient, &corev1.Service{}, "backend-listener", namespace, timeout, poll) + rvs["svc/backend-listener-nlb"] = testutil.GetResourceVersion( + k8sClient, &corev1.Service{}, "backend-listener-nlb", namespace, timeout, poll) rvs["deployment/backend-worker-canary"] = testutil.GetResourceVersion( k8sClient, &appsv1.Deployment{}, "backend-worker-canary", namespace, timeout, poll) @@ -634,8 +635,8 @@ var _ = Describe("Backend controller", func() { svc := &corev1.Service{} By("removing the backend-listener service deployment label selector", (&testutil.ExpectedResource{ - Name: "backend-listener", Namespace: namespace, - LastVersion: rvs["svc/backend-listener"], + Name: "backend-listener-nlb", Namespace: namespace, + LastVersion: rvs["svc/backend-listener-nlb"], }).Assert(k8sClient, svc, timeout, poll)) Expect(svc.Spec.Selector).ToNot(HaveKey("deployment")) diff --git a/controllers/echoapi_controller.go b/controllers/echoapi_controller.go index 19fedebb..afeb4010 100644 --- a/controllers/echoapi_controller.go +++ b/controllers/echoapi_controller.go @@ -18,12 +18,17 @@ package controllers import ( "context" + "strings" "github.com/3scale-ops/basereconciler/reconciler" "github.com/3scale-ops/basereconciler/util" saasv1alpha1 "github.com/3scale-ops/saas-operator/api/v1alpha1" "github.com/3scale-ops/saas-operator/pkg/generators/echoapi" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) // EchoAPIReconciler reconciles a EchoAPI object @@ -45,7 +50,7 @@ type EchoAPIReconciler struct { // move the current state of the cluster closer to the desired state. func (r *EchoAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - ctx, _ = r.Logger(ctx, "name", req.Name, "namespace", req.Namespace) + ctx, logger := r.Logger(ctx, "name", req.Name, "namespace", req.Namespace) instance := &saasv1alpha1.EchoAPI{} result := r.ManageResourceLifecycle(ctx, req, instance, reconciler.WithInMemoryInitializationFunc(util.ResourceDefaulter(instance))) @@ -54,6 +59,38 @@ func (r *EchoAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } gen := echoapi.NewGenerator(instance.GetName(), instance.GetNamespace(), instance.Spec) + + // Upgrade NLBs managed by nlb-helper-operator + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: gen.GetComponent(), Namespace: req.Namespace}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err == nil { + // 1. Update parent resource with NLB name for NLB adoption + // 2. Add termination protection + nlbName := strings.Split(strings.Split(svc.Status.LoadBalancer.Ingress[0].Hostname, ".")[0], "-")[0] + if instance.Spec.LoadBalancer.LoadBalancerName == nil || + *instance.Spec.LoadBalancer.LoadBalancerName != nlbName { + patch := client.MergeFrom(instance.DeepCopy()) + instance.Spec.LoadBalancer.LoadBalancerName = &nlbName + instance.Spec.LoadBalancer.TerminationProtection = util.Pointer(true) + if err := r.Client.Patch(ctx, instance, patch); err != nil { + return ctrl.Result{}, err + } + logger.Info("resource patched", "kind", "EchoAPI", "key", req) + } + + // 3. Abandon old Service resource + if svc.GetOwnerReferences() != nil || svc.GetAnnotations()["service.beta.kubernetes.io/aws-load-balancer-type"] != "external" { + svc.ObjectMeta.OwnerReferences = nil + svc.ObjectMeta.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"] = "external" + if err := r.Client.Update(ctx, svc); err != nil { + return ctrl.Result{}, err + } + logger.Info("resource abandoned", "kind", "Service", "key", req) + } + + } else if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + resources, err := gen.Resources() if err != nil { return ctrl.Result{}, err diff --git a/controllers/echoapi_controller_suite_test.go b/controllers/echoapi_controller_suite_test.go index b44e8dab..4e351f6a 100644 --- a/controllers/echoapi_controller_suite_test.go +++ b/controllers/echoapi_controller_suite_test.go @@ -93,8 +93,8 @@ var _ = Describe("EchoAPI controller", func() { Expect(dep.Spec.Template.Spec.Volumes).To(HaveLen(0)) svc := &corev1.Service{} - By("deploying an echo-api service", - (&testutil.ExpectedResource{Name: "echo-api", Namespace: namespace}). + By("deploying an echo-api-nlb service", + (&testutil.ExpectedResource{Name: "echo-api-nlb", Namespace: namespace}). Assert(k8sClient, svc, timeout, poll)) Expect(svc.Spec.Selector["deployment"]).To(Equal("echo-api")) diff --git a/controllers/upgrade_controller.go b/controllers/upgrade_controller.go new file mode 100644 index 00000000..cda051c4 --- /dev/null +++ b/controllers/upgrade_controller.go @@ -0,0 +1,165 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + hrp://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "reflect" + "strings" + "sync" + + "github.com/3scale-ops/basereconciler/reconciler" + "github.com/3scale-ops/basereconciler/util" + saasv1alpha1 "github.com/3scale-ops/saas-operator/api/v1alpha1" + "github.com/3scale-ops/saas-operator/pkg/generators/echoapi" + "github.com/davecgh/go-spew/spew" + "github.com/go-logr/logr" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type UpgradeTracker struct { + items []trackUpgrade + mu sync.Mutex +} + +func NewUpgradeTracker() *UpgradeTracker { + return &UpgradeTracker{ + items: []trackUpgrade{}, + } +} + +type trackUpgrade struct { + ref corev1.ObjectReference + upgraded bool +} + +func (r *UpgradeTracker) track(ref corev1.ObjectReference) { + if !lo.ContainsBy(r.items, func(x trackUpgrade) bool { + return reflect.DeepEqual(x.ref, ref) + }) { + r.mu.Lock() + defer r.mu.Unlock() + if r.items == nil { + r.items = []trackUpgrade{} + } + r.items = append(r.items, trackUpgrade{ + ref: ref, + upgraded: false, + }) + } +} + +func (r *UpgradeTracker) markUpgraded(ref corev1.ObjectReference) { + if _, index, ok := lo.FindIndexOf(r.items, func(x trackUpgrade) bool { + return reflect.DeepEqual(x.ref, ref) + }); ok { + r.mu.Lock() + defer r.mu.Unlock() + r.items[index].upgraded = true + } +} + +func (r *UpgradeTracker) CheckUpgradeCompleted() bool { + spew.Dump(r.items) + for _, item := range r.items { + if !item.upgraded { + return false + } + } + return true +} + +type EchoAPIUpgradeController struct { + *reconciler.Reconciler + *UpgradeTracker +} + +// The UpgradeControllers run at the start of the manager before any other controller is started +func (r *EchoAPIUpgradeController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var logger logr.Logger + ctx, logger = r.Logger(ctx, "name", req.Name, "namespace", req.Namespace) + + // add the object to the upgrade list + ref := corev1.ObjectReference{ + Kind: "EchoAPI", + Namespace: req.Namespace, + Name: req.Name, + APIVersion: "saas.3scale.net/v1alpha1", + } + r.track(ref) + + // get the instance and initialize + instance := &saasv1alpha1.EchoAPI{} + result := r.ManageResourceLifecycle(ctx, req, instance, + reconciler.WithInMemoryInitializationFunc(util.ResourceDefaulter(instance))) + if result.ShouldReturn() { + return result.Values() + } + + gen := echoapi.NewGenerator(instance.GetName(), instance.GetNamespace(), instance.Spec) + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: gen.GetComponent(), Namespace: req.Namespace}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil { + if errors.IsNotFound(err) { + // Old service has been pruned so there's nothing to upgrade + // Mark upgrade as complete and return + r.markUpgraded(ref) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // 1. Update parent resource with NLB name for NLB adoption + // 2. Add termination protection + nlbName := strings.Split(svc.Status.LoadBalancer.Ingress[0].Hostname, ".")[0] + if instance.Spec.LoadBalancer.LoadBalancerName != &nlbName { + patch := client.MergeFrom(instance.DeepCopy()) + instance.Spec.LoadBalancer.LoadBalancerName = &nlbName + instance.Spec.LoadBalancer.TerminationProtection = util.Pointer(true) + if err := r.Client.Patch(ctx, instance, patch); err != nil { + return ctrl.Result{}, err + } + logger.Info("resource patched", "kind", "EchoAPI", "key", req) + } + + // 3. Abandon old Service resource + if svc.GetOwnerReferences() != nil || svc.GetAnnotations()["service.beta.kubernetes.io/aws-load-balancer-type"] != "external" { + svc.ObjectMeta.OwnerReferences = nil + svc.ObjectMeta.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"] = "external" + if err := r.Client.Update(ctx, svc); err != nil { + return ctrl.Result{}, err + } + logger.Info("resource abandoned", "kind", "Service", "key", req) + return ctrl.Result{}, nil + } + + logger.Info("resource upgraded", "kind", "EchoAPI", "key", req) + r.markUpgraded(ref) + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *EchoAPIUpgradeController) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named("echoapi_upgrade_controller"). + For(&saasv1alpha1.EchoAPI{}). + Complete(r) +} diff --git a/docs/api-reference/reference.asciidoc b/docs/api-reference/reference.asciidoc index afb38ead..0edd26f8 100644 --- a/docs/api-reference/reference.asciidoc +++ b/docs/api-reference/reference.asciidoc @@ -179,6 +179,7 @@ AssetsSpec has configuration to access assets in AWS s3 | *`accessKey`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | AWS access key | *`secretKey`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | AWS secret access key | *`host`* __string__ | Assets host (CDN) +| *`s3Endpoint`* __string__ | Assets custom S3 endpoint |=== @@ -1435,9 +1436,22 @@ SecretReference is a reference to a secret stored in some secrets engine | Field | Description | *`fromVault`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-vaultsecretreference[$$VaultSecretReference$$]__ | FromVault is a reference to a secret key/value stored in a Hashicorp Vault | *`override`* __string__ | Override allows to directly specify a string value. +| *`fromSeed`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-seedsecretreference[$$SeedSecretReference$$]__ | FromSeed will try to retrieve the secret value from the default seed Secret. |=== +[id="{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-seedsecretreference"] +==== SeedSecretReference + +SeedSecretReference represents options to retrieve the secret value from the default seed Secret. There are no configurable options at this point. + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$] +**** + + + [id="{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-segmentspec"] ==== SegmentSpec @@ -1670,6 +1684,26 @@ System is the Schema for the systems API |=== +[id="{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-systemapicastendpointsspec"] +==== SystemApicastEndpointsSpec + +ApicastSpec holds properties to configure Apicast endpoints + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-systemconfig[$$SystemConfig$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`stagingDomain`* __string__ | Apicast Staging endpoint +| *`productionDomain`* __string__ | Apicast Production endpoint +| *`cloudHostedRegistryURL`* __string__ | Policies registry URL for Apicast Cloud Hosteed +| *`selfManagedRegistryURL`* __string__ | Policies registry URL for Apicast Self Managed (on-prem) +|=== + + [id="{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-systemappspec"] ==== SystemAppSpec @@ -1741,15 +1775,15 @@ SystemConfig holds configuration for SystemApp component | *`configFilesSecret`* __string__ | Secret containging system configuration files to be mounted in the pods | *`externalSecret`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-externalsecret[$$ExternalSecret$$]__ | External Secret common configuration | *`databaseDSN`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | DSN of system's main database -| *`eventsSharedSecret`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | EventsSharedSecret +| *`eventsSharedSecret`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | EventsSharedSecret is a password that protects System's event hooks endpoint. | *`recaptcha`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-systemrecaptchaspec[$$SystemRecaptchaSpec$$]__ | Holds recaptcha configuration options -| *`secretKeyBase`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | SecretKeyBase +| *`secretKeyBase`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | SecretKeyBase: https://api.rubyonrails.org/classes/Rails/Application.html#method-i-secret_key_base You can generate one random key using 'bundle exec rake secret' | *`accessCode`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | AccessCode to protect admin urls | *`segment`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-segmentspec[$$SegmentSpec$$]__ | Options for Segment integration | *`github`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-githubspec[$$GithubSpec$$]__ | Options for Github integration | *`redhatCustomerPortal`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-redhatcustomerportalspec[$$RedHatCustomerPortalSpec$$]__ | Options for configuring RH Customer Portal integration | *`bugsnag`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-bugsnagspec[$$BugsnagSpec$$]__ | Options for configuring Bugsnag integration -| *`databaseSecret`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | Database secret +| *`databaseSecret`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-secretreference[$$SecretReference$$]__ | DatabaseSecret is a site key stored off-database for improved more secure password hashing See https://github.com/3scale/porta/blob/ae498814cef3d856613f60d29330882fa870271d/config/initializers/site_keys.rb#L2-L19 | *`memcachedServers`* __string__ | Memcached servers | *`redis`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-redisspec[$$RedisSpec$$]__ | Redis configuration options | *`smtp`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-smtpspec[$$SMTPSpec$$]__ | SMTP configuration options @@ -1757,6 +1791,7 @@ SystemConfig holds configuration for SystemApp component | *`zync`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-systemzyncspec[$$SystemZyncSpec$$]__ | Zync has configuration options for system to contact zync | *`backend`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-systembackendspec[$$SystemBackendSpec$$]__ | Backend has configuration options for system to contact backend | *`assets`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-assetsspec[$$AssetsSpec$$]__ | Assets has configuration to access assets in AWS s3 +| *`apicast`* __xref:{anchor_prefix}-github-com-3scale-ops-saas-operator-api-v1alpha1-systemapicastendpointsspec[$$SystemApicastEndpointsSpec$$]__ | Apicast can be used to pass down apicast endpoints configuration |=== diff --git a/pkg/generators/backend/services.go b/pkg/generators/backend/services.go index 319d94f8..3f10ec1f 100644 --- a/pkg/generators/backend/services.go +++ b/pkg/generators/backend/services.go @@ -10,7 +10,7 @@ import ( func (gen *ListenerGenerator) service() *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: gen.GetComponent(), + Name: gen.GetComponent() + "-nlb", Annotations: service.NLBServiceAnnotations(*gen.ListenerSpec.LoadBalancer, gen.ListenerSpec.Endpoint.DNS), }, Spec: corev1.ServiceSpec{ diff --git a/pkg/generators/echoapi/service.go b/pkg/generators/echoapi/service.go index b78f0637..22890515 100644 --- a/pkg/generators/echoapi/service.go +++ b/pkg/generators/echoapi/service.go @@ -10,7 +10,7 @@ import ( func (gen *Generator) service() *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: gen.GetComponent(), + Name: gen.GetComponent() + "-nlb", Annotations: service.NLBServiceAnnotations(*gen.Spec.LoadBalancer, gen.Spec.Endpoint.DNS), }, Spec: corev1.ServiceSpec{ diff --git a/pkg/resource_builders/service/util.go b/pkg/resource_builders/service/util.go index 7dc106fb..34d19ef9 100644 --- a/pkg/resource_builders/service/util.go +++ b/pkg/resource_builders/service/util.go @@ -33,18 +33,31 @@ func ELBServiceAnnotations(cfg saasv1alpha1.LoadBalancerSpec, hostnames []string // NLBServiceAnnotations returns annotations for services exposed through AWS Network LoadBalancers func NLBServiceAnnotations(cfg saasv1alpha1.NLBLoadBalancerSpec, hostnames []string) map[string]string { annotations := map[string]string{ - "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", - "external-dns.alpha.kubernetes.io/hostname": strings.Join(hostnames, ","), - "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled": fmt.Sprintf("%t", *cfg.CrossZoneLoadBalancingEnabled), + "external-dns.alpha.kubernetes.io/hostname": strings.Join(hostnames, ","), + "service.beta.kubernetes.io/aws-load-balancer-type": "external", } - if *cfg.ProxyProtocol { - annotations["aws-nlb-helper.3scale.net/enable-targetgroups-proxy-protocol"] = "true" + annotations["service.beta.kubernetes.io/aws-load-balancer-proxy-protocol"] = "*" } - if len(cfg.EIPAllocations) != 0 { annotations["service.beta.kubernetes.io/aws-load-balancer-eip-allocations"] = strings.Join(cfg.EIPAllocations, ",") } + if cfg.LoadBalancerName != nil { + annotations["service.beta.kubernetes.io/aws-load-balancer-name"] = *cfg.LoadBalancerName + } + + attributes := []string{} + if *cfg.CrossZoneLoadBalancingEnabled { + attributes = append(attributes, "load_balancing.cross_zone.enabled=true") + } else { + attributes = append(attributes, "load_balancing.cross_zone.enabled=false") + } + if *cfg.TerminationProtection { + attributes = append(attributes, "deletion_protection.enabled=true") + } else { + attributes = append(attributes, "deletion_protection.enabled=false") + } + annotations["service.beta.kubernetes.io/aws-load-balancer-attributes"] = strings.Join(attributes, ",") return annotations } diff --git a/pkg/util/k8s.go b/pkg/util/k8s.go new file mode 100644 index 00000000..93ad8acc --- /dev/null +++ b/pkg/util/k8s.go @@ -0,0 +1,14 @@ +package util + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func IsOwner(owner, owned client.Object) bool { + for _, ownerRef := range owned.GetOwnerReferences() { + if ownerRef.Name == owner.GetName() && ownerRef.UID == owner.GetUID() && ownerRef.Kind == owner.GetObjectKind().GroupVersionKind().Kind { + return true + } + } + return false +} diff --git a/pkg/version/version.go b/pkg/version/version.go index 08394576..2580ed1d 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,7 +1,7 @@ package version const ( - version string = "v0.23.0-alpha.9" + version string = "v0.24.0-alpha.1" ) // Current returns the current marin3r operator version